0.01 version code

.gitignore

@@ -1,6 +1,5 @@
 # Binaries for programs and plugins
 *.exe
-*.exe~
 *.dll
 *.so
 *.dylib
@@ -10,3 +9,9 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+.git/
+.idea/

README.md

@@ -1,2 +1,8 @@
-# edge-rule-engine
-Lightweight IoT Rule Engine for Edge
+# edge_rule_engine
+
+#### Introduction
+A SQL based lightweight IoT streaming rule engine running at resource constrained edge devices.
+- SQL based, easy to use
+- Native run with small overhead 
+- Capability of consuming different source (preferred is MQTT)
+

common/util.go

@@ -0,0 +1,178 @@
+package common
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/dgraph-io/badger"
+	"github.com/sirupsen/logrus"
+	"os"
+	"time"
+)
+
+const LogLocation = "stream.log"
+
+var (
+	Log *logrus.Logger
+	Env string
+
+	logFile *os.File
+)
+
+type logRedirect struct {
+
+}
+
+func (l *logRedirect) Errorf(f string, v ...interface{}) {
+	Log.Error(fmt.Sprintf(f, v...))
+}
+
+func (l *logRedirect) Infof(f string, v ...interface{}) {
+	Log.Info(fmt.Sprintf(f, v...))
+}
+
+func (l *logRedirect) Warningf(f string, v ...interface{}) {
+	Log.Warning(fmt.Sprintf(f, v...))
+}
+
+func (l *logRedirect) Debugf(f string, v ...interface{}) {
+	Log.Debug(fmt.Sprintf(f, v...))
+}
+
+func init(){
+	flag.StringVar(&Env, "env", "dev", "set environment to prod or test")
+	flag.Parse()
+	Log = logrus.New()
+	if Env == "prod"{
+		logFile, err := os.OpenFile(LogLocation, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			Log.Out = logFile
+		} else {
+			Log.Infof("Failed to log to file, using default stderr")
+		}
+	}
+}
+
+func DbOpen(dir string) (*badger.DB, error) {
+	opts := badger.DefaultOptions
+	opts.Dir = dir
+	opts.ValueDir = dir
+	opts.Logger = &logRedirect{}
+	db, err := badger.Open(opts)
+	return db, err
+}
+
+func DbClose(db *badger.DB) error {
+	return db.Close()
+}
+
+func DbSet(db *badger.DB, key string, value string) error {
+
+	err := db.Update(func(txn *badger.Txn) error {
+		_, err := txn.Get([]byte(key))
+		//key not found
+		if err != nil {
+			err = txn.Set([]byte(key), []byte(value))
+		}else{
+			err = errors.New(fmt.Sprintf("key %s already exist, delete it before creating a new one", key))
+		}
+
+		return err
+	})
+	return err
+}
+
+func DbGet(db *badger.DB, key string) (value string, err error) {
+	err = db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get([]byte(key))
+		if err != nil {
+			return err
+		}
+
+		err = item.Value(func(val []byte) error {
+			value = string(val)
+			return nil
+		})
+		return err
+	})
+
+	return
+}
+
+func DbDelete(db *badger.DB, key string) error {
+	err := db.Update(func(txn *badger.Txn) error {
+		_, err := txn.Get([]byte(key))
+		//key not found
+		if err != nil {
+			return err
+		}else{
+			err = txn.Delete([]byte(key))
+		}
+		return err
+	})
+	return err
+}
+
+func DbKeys(db *badger.DB) (keys []string, err error) {
+	err = db.View(func(txn *badger.Txn) error {
+		opts := badger.DefaultIteratorOptions
+		opts.PrefetchSize = 10
+		it := txn.NewIterator(opts)
+		defer it.Close()
+		for it.Rewind(); it.Valid(); it.Next() {
+			item := it.Item()
+			k := item.Key()
+			keys = append(keys, string(k))
+		}
+		return nil
+	})
+	return
+}
+
+func PrintMap(m map[string]string, buff *bytes.Buffer) {
+
+	for k, v := range m {
+		buff.WriteString(fmt.Sprintf("%s: %s\n", k, v))
+	}
+}
+
+func CloseLogger(){
+	if logFile != nil {
+		logFile.Close()
+	}
+}
+
+func GetConfLoc()(string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	confDir := dir + "/conf/"
+	if _, err := os.Stat(confDir); os.IsNotExist(err) {
+		return "", err
+	}
+
+	return confDir, nil
+}
+
+
+func GetDataLoc() (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	dataDir := dir + "/data/"
+
+	if _, err := os.Stat(dataDir); os.IsNotExist(err) {
+		if err := os.Mkdir(dataDir, os.ModePerm); err != nil {
+			return "", fmt.Errorf("Find error %s when trying to locate xstream data folder.\n", err)
+		}
+	}
+
+	return dataDir, nil
+}
+
+func TimeToUnixMilli(time time.Time) int64 {
+	return time.UnixNano() / 1e6;
+}

docs/json_expr.md

@@ -0,0 +1,158 @@
+# Sample data
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44},
+    {"first": "Roger", "last": "Craig", "age": 68},
+    {"first": "Jane", "last": "Murphy", "age": 47}
+  ],
+    "followers": {
+        "Group1": [
+		    {"first": "John", "last": "Shavor", "age": 22},
+		    {"first": "Ken", "last": "Miller", "age": 33}
+        ],
+        "Group2": [
+            {"first": "Alice", "last": "Murphy", "age": 33},
+		    {"first": "Brian", "last": "Craig", "age": 44}
+        ]
+    }
+   "ops": {
+   	"functionA": {"numArgs": 2},
+    "functionB": {"numArgs": 3},
+    "functionC": {"variadic": true}
+  }
+}
+```
+
+# Basic expressions
+
+## Identifier 
+
+Source Dereference (`.`) The source dereference operator can be used to specify columns by dereferencing the source stream or table. The ``->`` dereference selects a key in a nested JSON object.
+
+```
+SELECT demo.age FROM demo
+{"age" : 37}
+```
+
+
+
+```
+SELECT demo.name->first FROM demo
+{"first" : "Tom"}
+```
+
+
+
+```
+SELECT name->first AS fname FROM demo
+{"fname": "Tom"}
+```
+
+## Index expression
+
+Index Expressions allow you to select a specific element in a list. It should look similar to array access in common programming languages. Indexing is 0 based.
+
+```
+SELECT children FROM demo
+{
+    "children": ["Sara","Alex","Jack"]
+}
+```
+
+
+
+```
+SELECT children[0] FROM demo
+{
+    "children": "Sara"
+}
+
+SELECT d.friends[0]->last FROM demo AS d
+{
+    "last" : "Murphy"
+}
+```
+
+# Slicing
+
+Slices allow you to select a contiguous subset of an array. 
+
+``field[from:to]`` If from is not specified, then it means start from the 1st element of array; If to is not specified, then it means end with the last element of array.
+
+```
+SELECT children[0:1] FROM demo
+{
+    "children": ["Sara","Alex"]
+}
+```
+
+
+
+```
+SELECT children[:] FROM demo == SELECT children FROM demo
+{
+    "children": ["Sara","Alex","Jack"]
+}
+```
+
+
+
+```
+SELECT children[:1] FROM demo
+{
+    "children": ["Sara","Alex"]
+}
+```
+
+
+
+```
+SELECT followers->Group1[:1]->first FROM demo
+
+{
+    "first": ["John","Alice"]
+}
+```
+
+
+
+# *Projections*
+
+<!--Do we need to support this?-->
+
+## List & Slice projections
+
+A wildcard expression creates a list projection, which is a projection over a JSON array. 
+
+```
+SELECT demo.friends[*]->first FROM demo
+{
+    "first": ["Dale", "Roger", "Jane"]
+}
+```
+
+
+
+```
+SELECT friends[:1]->first FROM demo
+{
+    "first": ["Dale", "Roger"]
+}
+```
+
+## Object projections
+
+
+
+```
+SELECT ops->*->numArgs FROM demo
+
+{ "numArgs" : [2, 3] }
+```
+

docs/streams.md

@@ -0,0 +1,184 @@
+## Stream specs 
+
+
+### Data types
+
+Refer to [Azure IoT](https://docs.microsoft.com/en-us/stream-analytics-query/data-types-azure-stream-analytics), boolean type is cast to int.
+
+| #    | Data type | Description                                                  |
+| ---- | --------- | ------------------------------------------------------------ |
+| 1    | bigint    |                                                              |
+| 2    | float     |                                                              |
+| 3    | string    |                                                              |
+| 4    | datetime  | Need to specify the date format?? Such as "yyyy-MM-dd"       |
+| 5    | boolean   |                                                              |
+| 6    | array     | The array type, can be any types from simple data or struct type (#1 - #5, and #7). |
+| 7    | struct    | The complex type.                                            |
+
+
+
+### Language definitions
+
+#### CREATE STREAM
+
+```sql
+CREATE STREAM   
+    stream_name   
+    ( column_name <data_type> [ ,...n ] )
+    WITH ( property_name = expression [, ...] );
+```
+
+**The supported property names.**
+
+| Property name | Optional | Description                                                  |
+| ------------- | -------- | ------------------------------------------------------------ |
+| DATASOURCE | false    | The topic names list if it's a MQTT data source. |
+| FORMAT        | false    | json or Avro.<br />Currently, we just support the JSON type ? |
+| KEY           | true     | It will be used in future for GROUP BY statements ??         |
+| TYPE     | false    | Is it requried in future if more & more sources are supported? By default, it would be MQTT type. |
+| StrictValidation     | false    | To control validation behavior of message field against stream schema. |
+| KEY_CONF | false | If additional configuration items are requied to be configured, then specify the config key here.<br />XStream currently propose yaml file format. |
+
+**Introduction for StrictValidation**
+
+``` 
+The value of StrictValidation can be true or false.
+1) True: Drop the message if the message  is not satisfy with the stream definition.
+2) False: Keep the message, but fill the missing field with default empty value.
+
+bigint: 0
+float: 0.0
+string: ""
+datetime: ??
+boolean: false
+array: zero length array
+struct: null value
+```
+
+Example 1,
+
+```sql
+CREATE STREAM demo (
+		USERID BIGINT,
+		FIRST_NAME STRING,
+		LAST_NAME STRING,
+		NICKNAMES ARRAY(STRING),
+		Gender BOOLEAN,
+		ADDRESS STRUCT(STREET_NAME STRING, NUMBER BIGINT),
+	) WITH (datasource="topics:test/, demo/test", FORMAT="AVRO", KEY="USERID", KEY_CONF="democonf");
+```
+
+
+
+Example 2,
+
+```sql
+CREATE STREAM my_stream   
+    (id int, name string, score float)
+    WITH ( datasource = "topic/temperature", FORMAT = "json", KEY = "id");
+```
+
+
+
+The configuration of MQTT source is specified with yaml format, and the configuration file location is at ``$xstream/etc/mqtt_source.yaml``.  Below is the file format.
+
+```yaml
+#Global MQTT configurations
+QoS: 1
+Share-subscription: true
+Servers:
+  - 
+     tcp://127.0.0.1:1883
+#TODO: Other global configurations
+
+
+#Override the global configurations
+demo: #Conf_key
+  QoS: 0
+  Servers:
+    - 
+      tls://10.211.55.6:1883
+
+
+```
+
+#### DROP STREAM
+
+DROP the stream.
+
+```
+DROP STREAM my_stream
+```
+
+#### DESCRIBE STREAM
+
+Print the stream definition.
+
+```
+DESC STREAM my_stream
+
+Fields
+-----------
+id     int
+name   string
+score  float
+
+SOURCE: topic/temperature
+Format: json
+Key: id
+```
+
+#### EXPLAIN STREAM
+
+Print the detailed runtime infomation of the stream.
+
+```
+EXPLAIN STREAM my_stream
+```
+
+#### SHOW STREAMS
+
+Print all defined streams in system.
+
+```
+SHOW STREAMS
+
+my_stream, iot_stream
+```
+
+
+
+### A simple CLI
+
+A simple command line tool is implemented in ``stream/cli/main.go``. To build the command line tool, run command ``go install -x engine/xstream/cli``.
+
+#### Run sql to manage streams
+
+Run `cli stream` command, after `xstream >` prompt shown, enter stream related sql statements such as create, drop, describe, explain and show stream statements to execute.
+
+```bash
+cli stream
+xstream > CREATE STREAM sname (count bigint) WITH (source="users", FORMAT="AVRO", KEY="USERID"
+xstream > DESCRIBE STREAM sname
+...
+```
+
+
+#### Run query
+
+```bash
+cli query
+xstream > select USERID from demo;
+...
+```
+
+
+
+### Implementation
+
+##### How to save the stream definitions?
+
+Refer to below, a storage is required for saving the stream definitions.
+
+![stream_storage](resources/stream_storage.png)
+

docs/tutorial.md

@@ -0,0 +1,17 @@
+
+
+## Directory structure 
+
+Below is the installation directory structure after installing xstream. 
+
+```
+xstream_installed_dir
+  bin
+    cli
+  etc
+    mqtt_source.yaml
+  data
+  plugins
+  log
+```
+

xsql/expression_evaluator.go

@@ -0,0 +1,43 @@
+package xsql
+
+import (
+	"engine/common"
+	"github.com/buger/jsonparser"
+	"github.com/golang-collections/collections/stack"
+)
+
+type ExpressionEvaluator struct {
+	*ExpressionVisitorAdaptor
+	tuple    []byte
+	operands *stack.Stack
+}
+
+func newExpressionEvaluator(tuple []byte) *ExpressionEvaluator {
+	ee := &ExpressionEvaluator{tuple: tuple}
+	ee.operands = stack.New()
+	return ee
+}
+
+func (ee *ExpressionEvaluator) Visit(expr Node) Visitor {
+	ee.DoVisit(ee, expr)
+	return nil
+}
+
+func (ee *ExpressionEvaluator) VisitBinaryExpr(expr *BinaryExpr) {
+	Walk(ee, expr.LHS)
+	Walk(ee, expr.RHS)
+	ee.operands.Push(expr.OP)
+}
+
+func (ee *ExpressionEvaluator) VisitFieldRef(expr *FieldRef) {
+	//TODO GetXXX, how to determine the type
+	if fv, err := jsonparser.GetInt(ee.tuple, expr.Name); err != nil {
+		common.Log.Printf("Cannot find value in %s with field name %s.\n", string(ee.tuple), expr.Name)
+	} else {
+		ee.operands.Push(fv)
+	}
+}
+
+func (ee *ExpressionEvaluator) VisitIntegerLiteral(expr *IntegerLiteral) {
+	ee.operands.Push(expr.Val)
+}

xsql/expression_evaluator_test.go

@@ -0,0 +1,31 @@
+package xsql
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func TestEE(t *testing.T) {
+	//stmt, err := NewParser(strings.NewReader(`SELECT * FROM TBL AS t1 WHERE t1.a*2+3>25 AND t1.b='hello'`)).Parse()
+	stmt, err := NewParser(strings.NewReader(`SELECT abc FROM tbl WHERE abc*2+3 > 12 AND abc < 20`)).Parse()
+	if err != nil {
+		t.Errorf("%s.\n", err)
+		return
+	}
+
+	d := []byte(`{"abc":21, "def":"hello"}`)
+	ee := newExpressionEvaluator(d)
+	Walk(ee, stmt.Condition)
+
+	if ee.operands.Len() <= 0 {
+		t.Error("No operands evaluated")
+	} else {
+		for {
+			if ee.operands.Len() <= 0 {
+				break
+			}
+			fmt.Printf("%s\n", ee.operands.Pop())
+		}
+	}
+}

xsql/functions.go

@@ -0,0 +1,31 @@
+package xsql
+
+import (
+	"math"
+	"strings"
+)
+
+type FunctionValuer struct{}
+
+var _ CallValuer = FunctionValuer{}
+
+func (FunctionValuer) Value(key string) (interface{}, bool) {
+	return nil, false
+}
+
+func (FunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
+	lowerName := strings.ToLower(name)
+	switch lowerName {
+	case "round":
+		arg0 := args[0].(float64)
+		return math.Round(arg0), true
+	case "abs":
+		arg0 := args[0].(float64)
+		return math.Abs(arg0), true
+	case "pow":
+		arg0, arg1 := args[0].(float64), args[1].(int64)
+		return math.Pow(arg0, float64(arg1)), true
+	default:
+		return nil, false
+	}
+}

xsql/lexical.go

@@ -0,0 +1,652 @@
+package xsql
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"strings"
+)
+
+type Token int
+
+const (
+	// Special tokens
+	ILLEGAL Token = iota
+	EOF
+	WS
+	COMMENT
+
+	AS
+	// Literals
+	IDENT // main
+
+	INTEGER   // 12345
+	NUMBER    //12345.67
+	STRING    // "abc"
+	BADSTRING // "abc
+
+	operatorBeg
+	// ADD and the following are InfluxQL Operators
+	ADD         // +
+	SUB         // -
+	MUL         // *
+	DIV         // /
+	MOD         // %
+	BITWISE_AND // &
+	BITWISE_OR  // |
+	BITWISE_XOR // ^
+
+	AND // AND
+	OR  // OR
+
+	EQ  // =
+	NEQ // !=
+	LT  // <
+	LTE // <=
+	GT  // >
+	GTE // >=
+
+	SUBSET //[
+	ARROW //->
+
+	operatorEnd
+
+	// Misc characters
+	ASTERISK  // *
+	COMMA     // ,
+	LPAREN    // (
+	RPAREN    // )
+	LBRACKET //[
+	RBRACKET  //]
+	HASH      // #
+	DOT       // .
+	COLON	  //:
+	SEMICOLON //;
+
+	// Keywords
+	SELECT
+	FROM
+	JOIN
+	LEFT
+	INNER
+	ON
+	WHERE
+	GROUP
+	ORDER
+	BY
+	ASC
+	DESC
+
+	TRUE
+	FALSE
+
+	CREATE
+	DROP
+	EXPLAIN
+	DESCRIBE
+	SHOW
+	STREAM
+	STREAMS
+	WITH
+
+	XBIGINT
+	XFLOAT
+	XSTRING
+	XDATETIME
+	XBOOLEAN
+	XARRAY
+	XSTRUCT
+
+	DATASOURCE
+	KEY
+	FORMAT
+	CONF_KEY
+	TYPE
+	STRICT_VALIDATION
+
+	DD
+	HH
+	MI
+	SS
+	MS
+)
+
+var tokens = []string{
+	ILLEGAL: "ILLEGAL",
+	EOF:     "EOF",
+	AS:      "AS",
+	WS:      "WS",
+	IDENT:   "IDENT",
+	INTEGER: "INTEGER",
+	NUMBER:  "NUMBER",
+	STRING:  "STRING",
+
+	ADD:         "+",
+	SUB:         "-",
+	MUL:         "*",
+	DIV:         "/",
+	MOD:         "%",
+	BITWISE_AND: "&",
+	BITWISE_OR:  "|",
+	BITWISE_XOR: "^",
+
+	EQ:  "=",
+	NEQ: "!=",
+	LT:  "<",
+	LTE: "<=",
+	GT:  ">",
+	GTE: ">=",
+
+	ARROW: "->",
+
+	ASTERISK: "*",
+	COMMA:    ",",
+
+	LPAREN:    "(",
+	RPAREN:    ")",
+	LBRACKET:  "[",
+	RBRACKET:  "]",
+	HASH:      "#",
+	DOT:       ".",
+	SEMICOLON: ";",
+	COLON:     ":",
+
+	SELECT: "SELECT",
+	FROM:   "FROM",
+	JOIN:   "JOIN",
+	LEFT:   "LEFT",
+	INNER:  "INNER",
+	ON:     "ON",
+	WHERE:  "WHERE",
+	GROUP:  "GROUP",
+	ORDER:  "ORDER",
+	BY:     "BY",
+	ASC:    "ASC",
+	DESC:   "DESC",
+
+	CREATE:   "CREATE",
+	DROP:     "RROP",
+	EXPLAIN:  "EXPLAIN",
+	DESCRIBE: "DESCRIBE",
+	SHOW:     "SHOW",
+	STREAM:   "STREAM",
+	STREAMS:  "STREAMS",
+	WITH:     "WITH",
+
+	XBIGINT:   "BIGINT",
+	XFLOAT:    "FLOAT",
+	XSTRING:   "STRING",
+	XDATETIME: "DATETIME",
+	XBOOLEAN:  "BOOLEAN",
+	XARRAY:    "ARRAY",
+	XSTRUCT:   "STRUCT",
+
+	DATASOURCE:   "DATASOURCE",
+	KEY:      "KEY",
+	FORMAT:   "FORMAT",
+	CONF_KEY: "CONF_KEY",
+	TYPE: 	  "TYPE",
+	STRICT_VALIDATION: "STRICT_VALIDATION",
+
+	AND: "AND",
+	OR:  "OR",
+	TRUE: "TRUE",
+	FALSE: "FALSE",
+
+	DD: "DD",
+	HH: "HH",
+	MI: "MI",
+	SS: "SS",
+	MS: "MS",
+}
+
+func (tok Token) String() string {
+	if tok >= 0 && tok < Token(len(tokens)) {
+		return tokens[tok]
+	}
+	return ""
+}
+
+type Scanner struct {
+	r *bufio.Reader
+}
+
+func NewScanner(r io.Reader) *Scanner {
+	return &Scanner{r: bufio.NewReader(r)}
+}
+
+func (s *Scanner) Scan() (tok Token, lit string) {
+	ch := s.read()
+	if isWhiteSpace(ch) {
+		//s.unread()
+		return s.ScanWhiteSpace()
+	} else if isLetter(ch) {
+		s.unread()
+		return s.ScanIdent()
+	} else if isQuotation(ch) {
+		s.unread()
+		return s.ScanString()
+	} else if isDigit(ch) {
+		s.unread()
+		return s.ScanNumber(false, false)
+	}
+
+	switch ch {
+	case eof:
+		return EOF, tokens[EOF]
+	case '=':
+		return EQ, tokens[EQ]
+	case '!':
+		_, _ = s.ScanWhiteSpace()
+		if r := s.read(); r == '=' {
+			return NEQ, tokens[NEQ]
+		} else {
+			s.unread()
+		}
+		return EQ, tokens[EQ]
+	case '<':
+		_, _ = s.ScanWhiteSpace()
+		if r := s.read(); r == '=' {
+			return LTE, tokens[LTE]
+		} else {
+			s.unread()
+		}
+		return LT, tokens[LT]
+	case '>':
+		_, _ = s.ScanWhiteSpace()
+		if r := s.read(); r == '=' {
+			return GTE, tokens[GTE]
+		} else {
+			s.unread()
+		}
+		return GT, tokens[GT]
+	case '+':
+		return ADD, tokens[ADD]
+	case '-':
+		_, _ = s.ScanWhiteSpace()
+		if r := s.read(); r == '-' {
+			s.skipUntilNewline()
+			return COMMENT, ""
+		} else if (r == '>'){
+			return ARROW, tokens[ARROW]
+		} else if isDigit(r) {
+			s.unread()
+			return s.ScanNumber(false, true)
+		} else if r == '.' {
+			_, _ = s.ScanWhiteSpace()
+			if r1 := s.read(); isDigit(r1) {
+				s.unread()
+				return s.ScanNumber(true, true)
+			} else {
+				s.unread()
+			}
+			s.unread()
+		} else {
+			s.unread()
+		}
+		return SUB, tokens[SUB]
+	case '/':
+		_, _ = s.ScanWhiteSpace()
+		if r := s.read(); r == '*' {
+			if err := s.skipUntilEndComment(); err != nil {
+				return ILLEGAL, ""
+			}
+			return COMMENT, ""
+		} else {
+			s.unread()
+		}
+		return DIV, tokens[DIV]
+	case '.':
+		if r := s.read(); isDigit(r) {
+			s.unread()
+			return s.ScanNumber(true, false)
+		}
+		s.unread()
+		return DOT, tokens[DOT]
+	case '%':
+		return MOD, tokens[MOD]
+	case '&':
+		return BITWISE_AND, tokens[BITWISE_AND]
+	case '|':
+		return BITWISE_OR, tokens[BITWISE_OR]
+	case '^':
+		return BITWISE_XOR, tokens[BITWISE_XOR]
+	case '*':
+		return ASTERISK, tokens[ASTERISK]
+	case ',':
+		return COMMA, tokens[COMMA]
+	case '(':
+		return LPAREN, tokens[LPAREN]
+	case ')':
+		return RPAREN, tokens[RPAREN]
+	case '[':
+		return LBRACKET, tokens[LBRACKET]
+	case ']':
+		return RBRACKET, tokens[RBRACKET]
+	case ':':
+		return COLON, tokens[COLON]
+	case '#':
+		return HASH, tokens[HASH]
+	case ';':
+		return SEMICOLON, tokens[SEMICOLON]
+	}
+	return ILLEGAL, ""
+}
+
+func (s *Scanner) ScanIdent() (tok Token, lit string) {
+	var buf bytes.Buffer
+	buf.WriteRune(s.read())
+	for {
+		if ch := s.read(); ch == eof {
+			break
+		} else if !isLetter(ch) && !isDigit(ch) && ch != '_' {
+			s.unread()
+			break
+		} else {
+			buf.WriteRune(ch)
+		}
+	}
+
+	switch lit = strings.ToUpper(buf.String()); lit {
+	case "SELECT":
+		return SELECT, lit
+	case "AS":
+		return AS, lit
+	case "FROM":
+		return FROM, lit
+	case "WHERE":
+		return WHERE, lit
+	case "AND":
+		return AND, lit
+	case "OR":
+		return OR, lit
+	case "GROUP":
+		return GROUP, lit
+	case "ORDER":
+		return ORDER, lit
+	case "BY":
+		return BY, lit
+	case "DESC":
+		return DESC, lit
+	case "ASC":
+		return ASC, lit
+	case "INNER":
+		return INNER, lit
+	case "LEFT":
+		return LEFT, lit
+	case "JOIN":
+		return JOIN, lit
+	case "ON":
+		return ON, lit
+	case "CREATE":
+		return CREATE, lit
+	case "DROP":
+		return DROP, lit
+	case "EXPLAIN":
+		return EXPLAIN, lit
+	case "DESCRIBE":
+		return DESCRIBE, lit
+	case "SHOW":
+		return SHOW, lit
+	case "STREAM":
+		return STREAM, lit
+	case "STREAMS":
+		return STREAMS, lit
+	case "WITH":
+		return WITH, lit
+	case "BIGINT":
+		return XBIGINT, lit
+	case "FLOAT":
+		return XFLOAT, lit
+	case "DATETIME":
+		return XDATETIME, lit
+	case "STRING":
+		return XSTRING, lit
+	case "BOOLEAN":
+		return XBOOLEAN, lit
+	case "ARRAY":
+		return XARRAY, lit
+	case "STRUCT":
+		return XSTRUCT, lit
+	case "DATASOURCE":
+		return DATASOURCE, lit
+	case "KEY":
+		return KEY, lit
+	case "FORMAT":
+		return FORMAT, lit
+	case "CONF_KEY":
+		return CONF_KEY, lit
+	case "TYPE":
+		return TYPE, lit
+	case "TRUE":
+		return TRUE, lit
+	case "FALSE":
+		return FALSE, lit
+	case "STRICT_VALIDATION":
+		return STRICT_VALIDATION, lit
+	case "DD":
+		return DD, lit
+	case "HH":
+		return HH, lit
+	case "MI":
+		return MI, lit
+	case "SS":
+		return SS, lit
+	case "MS":
+		return MS, lit
+	}
+
+	return IDENT, buf.String()
+}
+
+func (s *Scanner) ScanString() (tok Token, lit string) {
+	var buf bytes.Buffer
+	_ = s.read()
+	for {
+		ch := s.read()
+		if ch == '"' {
+			break
+		} else if ch == eof {
+			return BADSTRING, buf.String()
+		} else {
+			buf.WriteRune(ch)
+		}
+	}
+	return STRING, buf.String()
+}
+
+func (s *Scanner) ScanDigit() (tok Token, lit string) {
+	var buf bytes.Buffer
+	ch := s.read()
+	buf.WriteRune(ch)
+	for {
+		if ch := s.read(); isDigit(ch) {
+			buf.WriteRune(ch)
+		} else {
+			s.unread()
+			break
+		}
+	}
+	return INTEGER, buf.String()
+}
+
+func (s *Scanner) ScanNumber(startWithDot bool, isNeg bool) (tok Token, lit string) {
+	var buf bytes.Buffer
+
+	if isNeg {
+		buf.WriteRune('-')
+	}
+
+	if startWithDot {
+		buf.WriteRune('.')
+	}
+
+	ch := s.read()
+	buf.WriteRune(ch)
+
+	isNum := false
+	for {
+		if ch := s.read(); isDigit(ch) {
+			buf.WriteRune(ch)
+		} else if ch == '.' {
+			isNum = true
+			buf.WriteRune(ch)
+		} else {
+			s.unread()
+			break
+		}
+	}
+	if isNum || startWithDot {
+		return NUMBER, buf.String()
+	} else {
+		return INTEGER, buf.String()
+	}
+}
+
+func (s *Scanner) skipUntilNewline() {
+	for {
+		if ch := s.read(); ch == '\n' || ch == eof {
+			return
+		}
+	}
+}
+
+func (s *Scanner) skipUntilEndComment() error {
+	for {
+		if ch1 := s.read(); ch1 == '*' {
+			// We might be at the end.
+		star:
+			ch2 := s.read()
+			if ch2 == '/' {
+				return nil
+			} else if ch2 == '*' {
+				// We are back in the state machine since we see a star.
+				goto star
+			} else if ch2 == eof {
+				return io.EOF
+			}
+		} else if ch1 == eof {
+			return io.EOF
+		}
+	}
+}
+
+func (s *Scanner) ScanWhiteSpace() (tok Token, lit string) {
+	var buf bytes.Buffer
+	for {
+		if ch := s.read(); ch == eof {
+			break
+		} else if !isWhiteSpace(ch) {
+			s.unread()
+			break
+		} else {
+			buf.WriteRune(ch)
+		}
+	}
+	return WS, buf.String()
+}
+
+func (s *Scanner) read() rune {
+	ch, _, err := s.r.ReadRune()
+	if err != nil {
+		return eof
+	}
+	return ch
+}
+
+func (s *Scanner) unread() {
+	_ = s.r.UnreadRune()
+}
+
+var eof = rune(0)
+
+func isWhiteSpace(r rune) bool {
+	return (r == ' ') || (r == '\t') || (r == '\r') || (r == '\n')
+}
+
+func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
+
+func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
+
+func isQuotation(ch rune) bool { return ch == '"' }
+
+func (tok Token) isOperator() bool { return (tok > operatorBeg && tok < operatorEnd) || tok == ASTERISK || tok == LBRACKET }
+
+func (tok Token) isTimeLiteral() bool { return (tok >= DD && tok <= MS) }
+
+func (tok Token) allowedSourceToken() bool {
+	return (tok == IDENT || tok == DIV || tok == HASH || tok == ADD)
+}
+
+//Allowed special field name token
+func (tok Token) allowedSFNToken() bool { return (tok == DOT) }
+
+func (tok Token) Precedence() int {
+	switch tok {
+	case OR:
+		return 1
+	case AND:
+		return 2
+	case EQ, NEQ, LT, LTE, GT, GTE:
+		return 3
+	case ADD, SUB, BITWISE_OR, BITWISE_XOR:
+		return 4
+	case MUL, DIV, MOD, BITWISE_AND, SUBSET, ARROW:
+		return 5
+	}
+	return 0
+}
+
+type DataType int
+
+const (
+	UNKNOWN DataType = iota
+	BIGINT
+	FLOAT
+	STRINGS
+	DATETIME
+	BOOLEAN
+	ARRAY
+	STRUCT
+)
+
+var dataTypes = []string{
+	BIGINT	: "bigint",
+	FLOAT	: "float",
+	STRINGS	: "string",
+	DATETIME: "datetime",
+	BOOLEAN	: "boolean",
+	ARRAY	: "array",
+	STRUCT	: "struct",
+}
+
+func (d DataType) isSimpleType() bool {
+	return d >= BIGINT && d <= BOOLEAN
+}
+
+func (d DataType) String() string {
+	if d >= 0 && d < DataType(len(dataTypes)) {
+		return dataTypes[d]
+	}
+	return ""
+}
+
+func getDataType(tok Token) DataType {
+	switch tok {
+	case XBIGINT:
+		return BIGINT
+	case XFLOAT:
+		return FLOAT
+	case XSTRING:
+		return STRINGS
+	case XDATETIME:
+		return DATETIME
+	case XBOOLEAN:
+		return BOOLEAN
+	case XARRAY:
+		return ARRAY
+	case XSTRUCT:
+		return STRUCT
+	}
+	return UNKNOWN
+}

xsql/parser.go

@@ -0,0 +1,928 @@
+package xsql
+
+import (
+	"fmt"
+	"github.com/golang-collections/collections/stack"
+	"io"
+	"strconv"
+	"strings"
+)
+
+type Parser struct {
+	s *Scanner
+
+	i   int // buffer index
+	n   int // buffer char count
+	buf [3]struct {
+		tok Token
+		lit string
+	}
+}
+
+func (p *Parser) parseCondition() (Expr, error) {
+	if tok, _ := p.scanIgnoreWhitespace(); tok != WHERE {
+		p.unscan()
+		return nil, nil
+	}
+	expr, err := p.ParseExpr()
+	if err != nil {
+		return nil, err
+	}
+	return expr, nil
+}
+
+func (p *Parser) scan() (tok Token, lit string) {
+	if p.n > 0 {
+		p.n--
+		return p.curr()
+	}
+
+	tok, lit = p.s.Scan()
+
+	if tok != WS && tok != COMMENT {
+		p.i = (p.i + 1) % len(p.buf)
+		buf := &p.buf[p.i]
+		buf.tok, buf.lit = tok, lit
+	}
+
+	return
+}
+
+func (p *Parser) curr() (Token, string) {
+	i := (p.i - p.n + len(p.buf)) % len(p.buf)
+	buf := &p.buf[i]
+	return buf.tok, buf.lit
+}
+
+func (p *Parser) scanIgnoreWhitespace() (tok Token, lit string) {
+	tok, lit = p.scan()
+
+	for {
+		if tok == WS || tok == COMMENT {
+			tok, lit = p.scan()
+		} else {
+			break
+		}
+	}
+	return tok, lit
+}
+
+func (p *Parser) unscan() { p.n++ }
+
+func NewParser(r io.Reader) *Parser {
+	return &Parser{s: NewScanner(r)}
+}
+
+func (p *Parser) ParseQueries() (SelectStatements, error) {
+	var stmts SelectStatements
+
+	if stmt, err := p.Parse(); err != nil {
+		return nil, err
+	} else {
+		stmts = append(stmts, *stmt)
+	}
+
+	for {
+		if tok, _ := p.scanIgnoreWhitespace(); tok == SEMICOLON {
+			if stmt, err := p.Parse(); err != nil {
+				return nil, err
+			} else {
+				if stmt != nil {
+					stmts = append(stmts, *stmt)
+				}
+			}
+		} else if tok == EOF {
+			break
+		}
+	}
+
+	return stmts, nil
+}
+
+func (p *Parser) Parse() (*SelectStatement, error) {
+	selects := &SelectStatement{}
+
+	if tok, lit := p.scanIgnoreWhitespace(); tok == EOF {
+		return nil, nil
+	} else if tok != SELECT {
+		return nil, fmt.Errorf("Found %q, Expected SELECT.\n", lit)
+	}
+
+	if fields, err := p.parseFields(); err != nil {
+		return nil, err
+	} else {
+		selects.Fields = fields
+	}
+
+	if src, err := p.parseSource(); err != nil {
+		return nil, err
+	} else {
+		selects.Sources = src
+	}
+
+	if joins, err := p.parseJoins(); err != nil {
+		return nil, err
+	} else {
+		selects.Joins = joins
+	}
+
+	if exp, err := p.parseCondition(); err != nil {
+		return nil, err
+	} else {
+		if exp != nil {
+			selects.Condition = exp
+		}
+	}
+
+	if dims, err := p.parseDimensions(); err != nil {
+		return nil, err
+	} else {
+		selects.Dimensions = dims
+	}
+
+	if sorts, err := p.parseSorts(); err != nil {
+		return nil, err
+	} else {
+		selects.SortFields = sorts
+	}
+
+	if tok, lit := p.scanIgnoreWhitespace(); tok == SEMICOLON {
+		p.unscan()
+		return selects, nil
+	} else if tok != EOF {
+		return nil, fmt.Errorf("found %q, expected EOF.", lit)
+	}
+
+	return selects, nil
+}
+
+func (p *Parser) parseSource() (Sources, error) {
+	var sources Sources
+	if tok, lit := p.scanIgnoreWhitespace(); tok != FROM {
+		return nil, fmt.Errorf("found %q, expected FROM.", lit)
+	}
+
+	if src, alias, err := p.parseSourceLiteral(); err != nil {
+		return nil, err
+	} else {
+		sources = append(sources, &Table{Name: src, Alias: alias})
+	}
+
+	return sources, nil
+}
+
+//TODO Current func has problems when the source includes white space.
+func (p *Parser) parseSourceLiteral() (string, string, error) {
+	var sourceSeg []string
+	var alias string
+	for {
+		//HASH, DIV & ADD token is specially support for MQTT topic name patterns.
+		if tok, lit := p.scanIgnoreWhitespace(); tok.allowedSourceToken() {
+			sourceSeg = append(sourceSeg, lit)
+			if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == AS {
+				if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == IDENT {
+					alias = lit2
+				} else {
+					return "", "", fmt.Errorf("found %q, expected JOIN key word.", lit)
+				}
+			} else if tok1.allowedSourceToken() {
+				sourceSeg = append(sourceSeg, lit1)
+			} else {
+				p.unscan()
+				break
+			}
+		} else {
+			p.unscan()
+			break
+		}
+	}
+	return strings.Join(sourceSeg, ""), alias, nil
+}
+
+func (p *Parser) parseFieldNameSections() ([]string, error) {
+	var fieldNameSects []string
+	for {
+		if tok, lit := p.scanIgnoreWhitespace(); tok == IDENT {
+			fieldNameSects = append(fieldNameSects, lit)
+			if tok1, _ := p.scanIgnoreWhitespace(); !tok1.allowedSFNToken() {
+				p.unscan()
+				break
+			}
+		} else {
+			p.unscan()
+			break
+		}
+	}
+	if len(fieldNameSects) == 0 {
+		return nil, fmt.Errorf("Cannot find any field name.\n")
+	} else if len(fieldNameSects) > 2 {
+		return nil, fmt.Errorf("Too many field names. Please use -> to reference keys in struct.\n")
+	}
+	return fieldNameSects, nil
+}
+
+func (p *Parser) parseJoins() (Joins, error) {
+	var joins Joins
+	for {
+		if tok, lit := p.scanIgnoreWhitespace(); tok == INNER || tok == LEFT {
+			if tok1, _ := p.scanIgnoreWhitespace(); tok1 == JOIN {
+				if j, err := p.ParseJoin(); err != nil {
+					return nil, err
+				} else {
+					if tok == INNER {
+						j.JoinType = INNER_JOIN
+					} else if tok == LEFT {
+						j.JoinType = LEFT_JOIN
+					}
+					joins = append(joins, *j)
+				}
+			} else {
+				return nil, fmt.Errorf("found %q, expected JOIN key word.", lit)
+			}
+		} else {
+			p.unscan()
+			if len(joins) > 0 {
+				return joins, nil
+			}
+			return nil, nil
+		}
+	}
+	return joins, nil
+}
+
+func (p *Parser) ParseJoin() (*Join, error) {
+	var j = &Join{}
+	if src, alias, err := p.parseSourceLiteral(); err != nil {
+		return nil, err
+	} else {
+		j.Name = src
+		j.Alias = alias
+		if tok1, _ := p.scanIgnoreWhitespace(); tok1 == ON {
+			if exp, err := p.ParseExpr(); err != nil {
+				return nil, err
+			} else {
+				j.Expr = exp
+			}
+		} else {
+			p.unscan()
+		}
+	}
+	return j, nil
+}
+
+func (p *Parser) parseDimensions() (Dimensions, error) {
+	var ds Dimensions
+	if t, _ := p.scanIgnoreWhitespace(); t == GROUP {
+		if t1, l1 := p.scanIgnoreWhitespace(); t1 == BY {
+			for {
+				if exp, err := p.ParseExpr(); err != nil {
+					return nil, err
+				} else {
+					d := Dimension{Expr: exp}
+					ds = append(ds, d)
+				}
+				if tok, _ := p.scanIgnoreWhitespace(); tok == COMMA {
+					continue
+				} else {
+					p.unscan()
+					break
+				}
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected BY statement.", l1)
+		}
+	} else {
+		p.unscan()
+	}
+	return ds, nil
+}
+
+func (p *Parser) parseSorts() (SortFields, error) {
+	var ss SortFields
+	if t, _ := p.scanIgnoreWhitespace(); t == ORDER {
+		if t1, l1 := p.scanIgnoreWhitespace(); t1 == BY {
+			for {
+				if t1, l1 = p.scanIgnoreWhitespace(); t1 == IDENT {
+					s := SortField{Ascending: true}
+
+					p.unscan()
+					if name, err := p.parseFieldNameSections(); err == nil {
+						s.Name = strings.Join(name, tokens[DOT])
+					} else {
+						return nil, err
+					}
+
+					if t2, _ := p.scanIgnoreWhitespace(); t2 == DESC {
+						s.Ascending = false
+						ss = append(ss, s)
+					} else if t2 == ASC {
+						ss = append(ss, s)
+					} else {
+						ss = append(ss, s)
+						p.unscan()
+						continue
+					}
+				} else if t1 == COMMA {
+					continue
+				} else {
+					p.unscan()
+					break
+				}
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected BY keyword.", l1)
+		}
+	} else {
+		p.unscan()
+	}
+
+	return ss, nil
+}
+
+func (p *Parser) parseFields() (Fields, error) {
+	var fields Fields
+
+	tok, _ := p.scanIgnoreWhitespace()
+	if tok == ASTERISK {
+		fields = append(fields, Field{AName: "", Expr: &Wildcard{Token: tok}})
+		return fields, nil
+	}
+	p.unscan()
+
+	for {
+		field, err := p.parseField()
+
+		if err != nil {
+			return nil, err
+		} else {
+			fields = append(fields, *field)
+		}
+
+		tok, _ = p.scanIgnoreWhitespace()
+		if tok != COMMA {
+			p.unscan()
+			break
+		}
+	}
+	return fields, nil
+}
+
+func (p *Parser) parseField() (*Field, error) {
+	field := &Field{}
+	if exp, err := p.ParseExpr(); err != nil {
+		return nil, err
+	} else {
+		if e, ok := exp.(*FieldRef); ok {
+			field.Name = e.Name
+		} else if e, ok := exp.(*Call); ok {
+			field.Name = e.Name
+		}
+		field.Expr = exp
+	}
+
+	if alias, err := p.parseAlias(); err != nil {
+		return nil, err
+	} else {
+		if alias != "" {
+			field.AName = alias
+		}
+	}
+
+	return field, nil
+}
+
+func (p *Parser) parseAlias() (string, error) {
+	tok, lit := p.scanIgnoreWhitespace()
+	if tok == AS {
+		if tok, lit = p.scanIgnoreWhitespace(); tok != IDENT {
+			return "", fmt.Errorf("found %q, expected as alias.", lit)
+		} else {
+			return lit, nil
+		}
+	}
+	p.unscan()
+	return "", nil
+}
+
+func (p *Parser) ParseExpr() (Expr, error) {
+	var err error
+	root := &BinaryExpr{}
+
+	root.RHS, err = p.parseUnaryExpr()
+	if err != nil {
+		return nil, err
+	}
+
+	for {
+		op, _ := p.scanIgnoreWhitespace()
+		if !op.isOperator() {
+			p.unscan()
+			return root.RHS, nil
+		} else if op == ASTERISK { //Change the asterisk to Mul token.
+			op = MUL
+		} else if op == LBRACKET { //LBRACKET is a special token, need to unscan
+			op = SUBSET
+			p.unscan()
+		}
+
+		var rhs Expr
+		if rhs, err = p.parseUnaryExpr(); err != nil {
+			return nil, err
+		}
+
+		for node := root; ; {
+			r, ok := node.RHS.(*BinaryExpr)
+			if !ok || r.OP.Precedence() >= op.Precedence() {
+				node.RHS = &BinaryExpr{LHS: node.RHS, RHS: rhs, OP: op}
+				break
+			}
+			node = r
+		}
+	}
+
+	return nil, nil
+}
+
+func (p *Parser) parseUnaryExpr() (Expr, error) {
+	if tok1, _ := p.scanIgnoreWhitespace(); tok1 == LPAREN {
+		expr, err := p.ParseExpr()
+		if err != nil {
+			return nil, err
+		}
+		// Expect an RPAREN at the end.
+		if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 != RPAREN {
+			return nil, fmt.Errorf("found %q, expected right paren.", lit2)
+		}
+
+		return &ParenExpr{Expr: expr}, nil
+	} else if tok1 == LBRACKET {
+		return p.parseBracketExpr()
+	}
+
+	p.unscan()
+
+	tok, lit := p.scanIgnoreWhitespace()
+	if tok == IDENT {
+		if tok1, _ := p.scanIgnoreWhitespace(); tok1 == LPAREN {
+			return p.parseCall(lit)
+		}
+		p.unscan() //Back the Lparen token
+		p.unscan() //Back the ident token
+		if n, err := p.parseFieldNameSections(); err != nil {
+			return nil, err
+		} else {
+			if len(n) == 2 {
+				return &FieldRef{StreamName: StreamName(n[0]), Name: n[1]}, nil
+			}
+			return &FieldRef{StreamName: StreamName(""), Name: n[0]}, nil
+		}
+	} else if tok == STRING {
+		return &StringLiteral{Val: lit}, nil
+	} else if tok == INTEGER {
+		val, _ := strconv.Atoi(lit)
+		return &IntegerLiteral{Val: val}, nil
+	} else if tok == NUMBER {
+		if v, err := strconv.ParseFloat(lit, 64); err != nil {
+			return nil, fmt.Errorf("found %q, invalid number value.", lit)
+		} else {
+			return &NumberLiteral{Val: v}, nil
+		}
+	} else if tok == TRUE || tok == FALSE {
+		if v, err := strconv.ParseBool(lit); err != nil {
+			return nil, fmt.Errorf("found %q, invalid boolean value.", lit)
+		} else {
+			return &BooleanLiteral{Val:v}, nil
+		}
+	} else if tok.isTimeLiteral() {
+		return &TimeLiteral{Val:tok}, nil
+	}
+
+	return nil, fmt.Errorf("found %q, expected expression.", lit)
+}
+
+func (p *Parser) parseBracketExpr() (Expr, error){
+	tok2, lit2 := p.scanIgnoreWhitespace()
+	if tok2 == RBRACKET {
+		//field[]
+		return &ColonExpr{Start:0, End:-1}, nil
+	} else if tok2 == INTEGER {
+		start, err := strconv.Atoi(lit2)
+		if err != nil {
+			return nil, fmt.Errorf("The start index %s is not an int value in bracket expression.", lit2)
+		}
+		if tok3, _ := p.scanIgnoreWhitespace(); tok3 == RBRACKET {
+			//Such as field[2]
+			return &IndexExpr{Index:start}, nil
+		} else if tok3 == COLON {
+			//Such as field[2:] or field[2:4]
+			return p.parseColonExpr(start)
+		}
+	} else if tok2 == COLON {
+		//Such as field[:3] or [:]
+		return p.parseColonExpr(0)
+	}
+	return nil, fmt.Errorf("Unexpected token %q. when parsing bracket expressions.", lit2)
+}
+
+func (p *Parser) parseColonExpr(start int) (Expr, error) {
+	tok, lit := p.scanIgnoreWhitespace()
+	if tok == INTEGER {
+		end, err := strconv.Atoi(lit)
+		if err != nil {
+			return nil, fmt.Errorf("The end index %s is not an int value in bracket expression.", lit)
+		}
+
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == RBRACKET {
+			return &ColonExpr{Start:start, End: end}, nil
+		} else {
+			return nil, fmt.Errorf("Found %q, expected right bracket.", lit1)
+		}
+	} else if tok == RBRACKET {
+		return &ColonExpr{Start:start, End: -1}, nil
+	}
+	return nil, fmt.Errorf("Found %q, expected right bracket.", lit)
+}
+
+func (p *Parser) parseAs(f *Field) (*Field, error) {
+	tok, lit := p.scanIgnoreWhitespace()
+	if tok != IDENT {
+		return nil, fmt.Errorf("found %q, expected as alias.", lit)
+	}
+	f.AName = lit
+	return f, nil
+}
+
+func (p *Parser) parseCall(name string) (Expr, error) {
+	var args []Expr
+	for {
+		if tok, _ := p.scanIgnoreWhitespace(); tok == RPAREN {
+			return &Call{Name: name, Args: args}, nil
+		} else {
+			p.unscan()
+		}
+		if exp, err := p.ParseExpr(); err != nil {
+			return nil, err
+		} else {
+			args = append(args, exp)
+		}
+
+		if tok, _ := p.scanIgnoreWhitespace(); tok != COMMA {
+			p.unscan()
+			break
+		}
+	}
+
+	if tok, lit := p.scanIgnoreWhitespace(); tok != RPAREN {
+		return nil, fmt.Errorf("found function call %q, expected ), but with %q.", name, lit)
+	}
+	if wt, error := validateWindows(name, args); wt == NOT_WINDOW {
+		return &Call{Name: name, Args: args}, nil
+	} else {
+		if error != nil {
+			return nil, error
+		}
+		return p.ConvertToWindows(wt, name, args)
+	}
+}
+
+func validateWindows(name string, args []Expr) (WindowType, error) {
+	fname := strings.ToLower(name)
+	switch fname {
+	case "tumblingwindow":
+		if err := validateWindow(fname, 2, args); err != nil {
+			return TUMBLING_WINDOW, err
+		}
+		return TUMBLING_WINDOW, nil
+	case "hoppingwindow":
+		if err := validateWindow(fname, 3, args); err != nil {
+			return HOPPING_WINDOW, err
+		}
+		return HOPPING_WINDOW, nil
+	case "sessionwindow":
+		if err := validateWindow(fname, 3, args); err != nil {
+			return SESSION_WINDOW, err
+		}
+		return SESSION_WINDOW, nil
+	case "slidingwindow":
+		if err := validateWindow(fname, 2, args); err != nil {
+			return SLIDING_WINDOW, err
+		}
+		return SLIDING_WINDOW, nil
+	}
+	return NOT_WINDOW, nil
+}
+
+func validateWindow(funcName string, expectLen int, args []Expr) (error) {
+	if len(args) != expectLen {
+		return fmt.Errorf("The arguments for %s should be %d.\n", funcName, expectLen)
+	}
+	if _, ok := args[0].(*TimeLiteral); ok {
+		return nil
+	} else {
+		return fmt.Errorf("The 1st argument for %s is expecting timer literal expression. One value of [dd|hh|mi|ss|ms].\n", funcName)
+	}
+}
+
+func (p *Parser) ConvertToWindows(wtype WindowType, name string, args []Expr) (*Windows, error) {
+	win := &Windows{WindowType:wtype}
+
+	win.Args = args
+	return win, nil
+}
+
+func (p *Parser) ParseCreateStreamStmt() (*StreamStmt, error) {
+	stmt := &StreamStmt{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == CREATE {
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == STREAM {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == IDENT {
+				stmt.Name = StreamName(lit2)
+				if fields, err := p.parseStreamFields(); err != nil {
+					return nil, err
+				} else {
+					stmt.StreamFields = fields
+				}
+				if opts, err := p.parseStreamOptions(); err != nil {
+					return nil, err
+				} else {
+					stmt.Options = opts
+				}
+				if tok3, lit3 := p.scanIgnoreWhitespace(); tok3 == SEMICOLON {
+					p.unscan()
+					return stmt, nil
+				} else if tok3 == EOF {
+					//Finish parsing create stream statement.
+					return stmt, nil
+				} else {
+					return nil, fmt.Errorf("found %q, expected semicolon or EOF.", lit3)
+				}
+
+			} else {
+				return nil, fmt.Errorf("found %q, expected stream name.", lit2)
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected keyword stream.", lit1)
+		}
+	} else {
+		p.unscan()
+		return nil, nil
+	}
+	return stmt, nil
+}
+
+func (p *Parser) parseShowStreamsStmt() (*ShowStreamsStatement, error) {
+	ss := &ShowStreamsStatement{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == SHOW {
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == STREAMS {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == EOF || tok2 == SEMICOLON {
+				return ss, nil
+			} else {
+				return nil, fmt.Errorf("found %q, expected semecolon or EOF.", lit2)
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected keyword streams.", lit1)
+		}
+	} else {
+		p.unscan()
+		return nil, nil
+	}
+	return ss, nil
+}
+
+func (p *Parser) parseDescribeStreamStmt() (*DescribeStreamStatement, error) {
+	dss := &DescribeStreamStatement{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == DESCRIBE {
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == STREAM {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == IDENT {
+				dss.Name = lit2
+				return dss, nil
+			} else {
+				return nil, fmt.Errorf("found %q, expected stream name.", lit2)
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected keyword stream.", lit1)
+		}
+	} else {
+		p.unscan()
+		return nil, nil
+	}
+}
+
+func (p *Parser) parseExplainStreamsStmt() (*ExplainStreamStatement, error) {
+	ess := &ExplainStreamStatement{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == EXPLAIN {
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == STREAM {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == IDENT {
+				ess.Name = lit2
+				return ess, nil
+			} else {
+				return nil, fmt.Errorf("found %q, expected stream name.", lit2)
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected keyword stream.", lit1)
+		}
+	} else {
+		p.unscan()
+		return nil, nil
+	}
+}
+
+func (p *Parser) parseDropStreamsStmt() (*DropStreamStatement, error) {
+	ess := &DropStreamStatement{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == DROP {
+		if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == STREAM {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == IDENT {
+				ess.Name = lit2
+				return ess, nil
+			} else {
+				return nil, fmt.Errorf("found %q, expected stream name.", lit2)
+			}
+		} else {
+			return nil, fmt.Errorf("found %q, expected keyword stream.", lit1)
+		}
+	} else {
+		p.unscan()
+		return nil, nil
+	}
+}
+
+func (p *Parser) parseStreamFields() (StreamFields, error) {
+	lStack := &stack.Stack{}
+	var fields StreamFields
+	if tok, lit := p.scanIgnoreWhitespace(); tok == LPAREN {
+		lStack.Push(lit)
+		for {
+			if f, err := p.parseStreamField(); err != nil {
+				return nil, err
+			} else {
+				fields = append(fields, *f)
+			}
+
+			if tok1, _ := p.scanIgnoreWhitespace(); tok1 == RPAREN {
+				lStack.Pop()
+				if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == WITH {
+					//Check the stack for LPAREN; If the stack for LPAREN is not zero, then it's not correct.
+					if lStack.Len() > 0 {
+						return nil, fmt.Errorf("Parenthesis is not matched.")
+					}
+					break
+				} else if tok2 == COMMA {
+					if lStack.Len() > 0 {
+						return nil, fmt.Errorf("Parenthesis is in create record type not matched.")
+					}
+					p.unscan()
+					break
+				} else if tok2 == RPAREN { //The nested type definition of ARRAY and Struct, such as "field ARRAY(STRUCT(f BIGINT))"
+					if lStack.Len() > 0 {
+						return nil, fmt.Errorf("Parenthesis is not matched.")
+					}
+					p.unscan()
+					break
+				} else {
+					if lStack.Len() == 0 {
+						return nil, fmt.Errorf("found %q, expected is with.", lit2)
+					}
+					p.unscan()
+				}
+			} else {
+				p.unscan()
+			}
+		}
+
+	} else {
+		return nil, fmt.Errorf("found %q, expected lparen after stream name.", lit)
+	}
+	return fields, nil
+}
+
+func (p *Parser) parseStreamField() (*StreamField, error) {
+	field := &StreamField{}
+	if tok, lit := p.scanIgnoreWhitespace(); tok == IDENT {
+		field.Name = lit
+		tok1, lit1 := p.scanIgnoreWhitespace()
+		if t := getDataType(tok1); t != UNKNOWN && t.isSimpleType() {
+			field.FieldType = &BasicType{Type: t}
+		} else if t == ARRAY {
+			if f, e := p.parseStreamArrayType(); e != nil {
+				return nil, e
+			} else {
+				field.FieldType = f
+			}
+		} else if t == STRUCT {
+			if f, e := p.parseStreamStructType(); e != nil {
+				return nil, e
+			} else {
+				field.FieldType = f
+			}
+		} else if t == UNKNOWN {
+			return nil, fmt.Errorf("found %q, expect valid stream field types(BIGINT | FLOAT | STRINGS | DATETIME | BOOLEAN | ARRAY | STRUCT).", lit1)
+		}
+
+		if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == COMMA {
+			//Just consume the comma.
+		} else if tok2 == RPAREN {
+			p.unscan()
+		} else {
+			return nil, fmt.Errorf("found %q, expect comma or rparen.", lit2)
+		}
+	} else {
+		return nil, fmt.Errorf("found %q, expect stream field name.", lit)
+	}
+	return field, nil
+}
+
+func (p *Parser) parseStreamArrayType() (FieldType, error) {
+	lStack := &stack.Stack{}
+	if tok, _ := p.scanIgnoreWhitespace(); tok == LPAREN {
+		lStack.Push(LPAREN)
+		tok1, lit1 := p.scanIgnoreWhitespace()
+		if t := getDataType(tok1); t != UNKNOWN && t.isSimpleType() {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == RPAREN {
+				lStack.Pop()
+				if lStack.Len() > 0 {
+					return nil, fmt.Errorf("Parenthesis is in array type not matched.")
+				}
+				return &ArrayType{Type: t}, nil
+			} else {
+				return nil, fmt.Errorf("found %q, expect rparen in array type definition.", lit2)
+			}
+		} else if tok1 == XSTRUCT {
+			if f, err := p.parseStreamStructType(); err != nil {
+				return nil, err
+			} else {
+				if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == RPAREN {
+					lStack.Pop()
+					if lStack.Len() > 0 {
+						return nil, fmt.Errorf("Parenthesis is in struct of array type %q not matched.", tok1)
+					}
+					return &ArrayType{Type: STRUCT, FieldType: f}, nil
+				} else {
+					return nil, fmt.Errorf("found %q, expect rparen in struct of array type definition.", lit2)
+				}
+			}
+		} else if tok1 == COMMA {
+			p.unscan()
+		} else {
+			return nil, fmt.Errorf("found %q, expect stream data types.", lit1)
+		}
+	} else {
+
+	}
+	return nil, nil
+}
+
+func (p *Parser) parseStreamStructType() (FieldType, error) {
+	rf := &RecType{}
+	if sfs, err := p.parseStreamFields(); err != nil {
+		return nil, err
+	} else {
+		if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == COMMA {
+			rf.StreamFields = sfs
+		} else if tok2 == RPAREN {
+			rf.StreamFields = sfs
+			p.unscan()
+		} else {
+			return nil, fmt.Errorf("found %q, expect comma in create stream record statement.", lit2)
+		}
+	}
+	return rf, nil
+}
+
+func (p *Parser) parseStreamOptions() (map[string]string, error) {
+	var opts map[string]string = make(map[string]string)
+	lStack := &stack.Stack{}
+	if tok, lit := p.scanIgnoreWhitespace(); tok == LPAREN {
+		lStack.Push(LPAREN)
+		for {
+			if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == DATASOURCE || tok1 == FORMAT || tok1 == KEY || tok1 == CONF_KEY || tok1 == STRICT_VALIDATION || tok1 == TYPE {
+				if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == EQ {
+					if tok3, lit3 := p.scanIgnoreWhitespace(); tok3 == STRING {
+						if tok1 == STRICT_VALIDATION {
+							if val := strings.ToUpper(lit3); (val != "TRUE") && (val != "FALSE") {
+								return nil, fmt.Errorf("found %q, expect TRUE/FALSE value in %s option.", lit3, tok1)
+							}
+						}
+						opts[lit1] = lit3
+					} else {
+						return nil, fmt.Errorf("found %q, expect string value in option.", lit3)
+					}
+				} else {
+					return nil, fmt.Errorf("found %q, expect equals(=) in options.", lit2)
+				}
+
+			} else if tok1 == COMMA {
+				continue
+			} else if tok1 == RPAREN {
+				if lStack.Pop(); lStack.Len() == 0 {
+					break
+				} else {
+					return nil, fmt.Errorf("Parenthesis is not matched in options definition.")
+				}
+			} else {
+				return nil, fmt.Errorf("found %q, unknown option keys(DATASOURCE|FORMAT|KEY|CONF_KEY|STRICT_VALIDATION|TYPE).", lit1)
+			}
+		}
+	} else {
+		return nil, fmt.Errorf("found %q, expect stream options.", lit)
+	}
+	return opts, nil
+}

xsql/plans/filter_operator.go

@@ -0,0 +1,33 @@
+package plans
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+)
+
+type FilterPlan struct {
+	Condition xsql.Expr
+}
+
+func (p *FilterPlan) Apply(ctx context.Context, data interface{}) interface{} {
+	log := common.Log
+	var input map[string]interface{}
+	if d, ok := data.(map[string]interface{}); !ok {
+		log.Errorf("Expect map[string]interface{} type.\n")
+		return nil
+	} else {
+		input = d
+	}
+	log.Infof("filterplan receive %s", input)
+	ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(xsql.MapValuer(input), &xsql.FunctionValuer{})}
+	result, ok := ve.Eval(p.Condition).(bool)
+	if ok {
+		if result {
+			return input
+		}
+	} else {
+		log.Errorf("invalid condition that returns non-bool value")
+	}
+	return nil
+}

xsql/plans/filter_test.go

@@ -0,0 +1,63 @@
+package plans
+
+import (
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestFilterPlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data map[string]interface{}
+		result interface{}
+	}{
+		{
+			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 AND abc < 20",
+			data: map[string]interface{}{
+				"a" : int64(6),
+			},
+			result: nil,
+		},
+
+		{
+			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 AND abc < 20",
+			data: map[string]interface{}{
+				"abc" : int64(6),
+			},
+			result: map[string]interface{}{
+				"abc" : int64(6),
+			},
+		},
+
+		{
+			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 OR def = \"hello\"",
+			data: map[string]interface{}{
+				"abc" : int64(34),
+				"def" : "hello",
+			},
+			result: map[string]interface{}{
+				"abc" : int64(34),
+				"def" : "hello",
+			},
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		pp := &FilterPlan{Condition:stmt.Condition}
+		result := pp.Apply(nil, tt.data)
+		if !reflect.DeepEqual(tt.result, result) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
+		}
+	}
+}

xsql/plans/join_operator.go

@@ -0,0 +1,40 @@
+package plans
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+)
+
+type JoinPlan struct {
+	Joins xsql.Joins
+}
+
+func (jp *JoinPlan) Apply(ctx context.Context, data interface{}) interface{} {
+	var log = common.Log
+	var input xsql.MultiEmitterTuples
+	if d, ok := data.(xsql.MultiEmitterTuples ); !ok {
+		log.Errorf("Expect MultiEmitterTuples type.\n")
+		return nil
+	} else {
+		input = d
+	}
+
+	result := xsql.MergedEmitterTupleSets{}
+
+	for _, join := range jp.Joins {
+		ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, &xsql.FunctionValuer{}), JoinType: join.JoinType}
+		v := ve.Eval(join.Expr)
+		if v1, ok := v.(xsql.MergedEmitterTupleSets); ok {
+			result = jp.mergeSet(v1, result)
+		}
+	}
+	return result
+}
+
+
+func (jp *JoinPlan) mergeSet(set1 xsql.MergedEmitterTupleSets, set2 xsql.MergedEmitterTupleSets) xsql.MergedEmitterTupleSets {
+	return set1
+}
+
+

xsql/plans/join_test.go

@@ -0,0 +1,714 @@
+package plans
+
+import (
+	"encoding/json"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestLeftJoinPlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data xsql.MultiEmitterTuples
+		result interface{}
+	}{
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 As s1 left join src2 as s2 on s1.id1 = s2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"s1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"s2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "s1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "s2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "s1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "s2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "s1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w2",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter: "src2",
+					Messages: nil,
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{} {},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:nil,
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1*2 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2*2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1", },},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : "v3",},},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.f1->cid = src2.f2->cid",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : str2Map(`{"cid" : 1, "name" : "tom1"}`), },
+						{ "id1" : 2, "f1" : str2Map(`{"cid" : 2, "name" : "mike1"}`), },
+						{ "id1" : 3, "f1" : str2Map(`{"cid" : 3, "name" : "alice1"}`), },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : str2Map(`{"cid" : 1, "name" : "tom2"}`),},
+						{ "id2" : 2, "f2" : str2Map(`{"cid" : 2, "name" : "mike2"}`), },
+						{ "id2" : 4, "f2" : str2Map(`{"cid" : 4, "name" : "alice2"}`), },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : str2Map(`{"cid" : 1, "name" : "tom1"}`), },},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : str2Map(`{"cid" : 1, "name" : "tom2"}`), },},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : str2Map(`{"cid" : 2, "name" : "mike1"}`), },},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : str2Map(`{"cid" : 2, "name" : "mike2"}`), },},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 3, "f1" : str2Map(`{"cid" : 3, "name" : "alice1"}`), },},
+					},
+				},
+			},
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		pp := &JoinPlan{Joins : stmt.Joins}
+		result := pp.Apply(nil, tt.data)
+		if !reflect.DeepEqual(tt.result, result) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
+		}
+	}
+}
+
+func TestInnerJoinPlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data xsql.MultiEmitterTuples
+		result interface{}
+	}{
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 As s1 inner join src2 as s2 on s1.id1 = s2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"s1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"s2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "s1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "s2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "s1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "s2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w2",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter: "src2",
+					Messages: nil,
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{} {},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{ },
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:nil,
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 1, "f2" : "w2", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1*2 = src2.id2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.id1 = src2.id2*2",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : "v1", },
+						{ "id1" : 2, "f1" : "v2", },
+						{ "id1" : 3, "f1" : "v3", },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : "w1", },
+						{ "id2" : 2, "f2" : "w2", },
+						{ "id2" : 4, "f2" : "w3", },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : "w1", },},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT id1 FROM src1 inner join src2 on src1.f1->cid = src2.f2->cid",
+			data: xsql.MultiEmitterTuples{
+				xsql.EmitterTuples{
+					Emitter:"src1",
+					Messages:[]map[string]interface{}{
+						{ "id1" : 1, "f1" : str2Map(`{"cid" : 1, "name" : "tom1"}`), },
+						{ "id1" : 2, "f1" : str2Map(`{"cid" : 2, "name" : "mike1"}`), },
+						{ "id1" : 3, "f1" : str2Map(`{"cid" : 3, "name" : "alice1"}`), },
+					},
+				},
+
+				xsql.EmitterTuples{
+					Emitter:"src2",
+					Messages:[]map[string]interface{}{
+						{ "id2" : 1, "f2" : str2Map(`{"cid" : 1, "name" : "tom2"}`),},
+						{ "id2" : 2, "f2" : str2Map(`{"cid" : 2, "name" : "mike2"}`), },
+						{ "id2" : 4, "f2" : str2Map(`{"cid" : 4, "name" : "alice2"}`), },
+					},
+				},
+			},
+			result: xsql.MergedEmitterTupleSets{
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 1, "f1" : str2Map(`{"cid" : 1, "name" : "tom1"}`), },},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 1, "f2" : str2Map(`{"cid" : 1, "name" : "tom2"}`), },},
+					},
+				},
+				xsql.MergedEmitterTuple{
+					MergedMessage: []xsql.EmitterTuple{
+						{Emitter: "src1", Message: map[string]interface{}{ "id1" : 2, "f1" : str2Map(`{"cid" : 2, "name" : "mike1"}`), },},
+						{Emitter: "src2", Message: map[string]interface{}{ "id2" : 2, "f2" : str2Map(`{"cid" : 2, "name" : "mike2"}`), },},
+					},
+				},
+			},
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		pp := &JoinPlan{Joins : stmt.Joins}
+		result := pp.Apply(nil, tt.data)
+		if !reflect.DeepEqual(tt.result, result) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
+		}
+	}
+}
+
+func str2Map(s string) map[string]interface{} {
+	var input map[string]interface{}
+	if err := json.Unmarshal([]byte(s), &input); err != nil {
+		fmt.Printf("Failed to parse the JSON data.\n")
+		return nil
+	}
+	return input
+}

xsql/plans/preprocessor.go

@@ -0,0 +1,217 @@
+package plans
+
+import (
+	"context"
+	"encoding/json"
+	"engine/common"
+	"engine/xsql"
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+type Preprocessor struct {
+	StreamStmt *xsql.StreamStmt
+}
+
+// data is a json string
+func (p *Preprocessor) Apply(ctx context.Context, data interface{}) interface{} {
+	log := common.Log
+	tuple, ok := data.(*xsql.Tuple)
+	if !ok {
+		log.Errorf("Expect tuple data type.\n")
+		return nil
+	}
+	var jsonData []byte
+	if d, ok := tuple.Message.([]byte); !ok {
+		log.Errorf("Expect byte array data type.\n")
+		return nil
+	} else {
+		jsonData = d
+	}
+	log.Infof("preprocessor receive %s", jsonData)
+	//The unmarshal type can only be bool, float64, string, []interface{}, map[string]interface{}, nil
+	jsonResult := make(map[string]interface{})
+
+	result := make(map[string]interface{})
+	if e := json.Unmarshal(jsonData, &jsonResult); e != nil {
+		log.Errorf("parse json string %s error: %s", jsonData, e)
+		return nil
+	}else{
+		for _, f := range p.StreamStmt.StreamFields {
+			if e = addRecField(f.FieldType, result, jsonResult, f.Name); e != nil{
+				log.Errorf("error in preprocessor: %s", e)
+				return nil
+			}
+		}
+		tuple.Message = result
+		return tuple
+	}
+}
+
+func addRecField(ft xsql.FieldType, r map[string]interface{}, j map[string]interface{}, p string) error {
+	if t, ok := j[p]; ok {
+		v := reflect.ValueOf(t)
+		jtype := v.Kind()
+		switch st := ft.(type) {
+		case *xsql.BasicType:
+			switch st.Type {
+			case xsql.UNKNOWN:
+				return errors.New(fmt.Sprintf("invalid data type unknown defined for %s, please check the stream definition", t))
+			case xsql.BIGINT:
+				if jtype == reflect.Float64{
+					r[p] = int(t.(float64))
+				}else{
+					return errors.New(fmt.Sprintf("invalid data type for %s, expect bigint but found %s", p, t))
+				}
+			case xsql.FLOAT:
+				if jtype == reflect.Float64{
+					r[p] = t.(float64)
+				}else{
+					return errors.New(fmt.Sprintf("invalid data type for %s, expect float but found %s", p, t))
+				}
+			case xsql.STRINGS:
+				if jtype == reflect.String{
+					r[p] = t.(string)
+				}else{
+					return errors.New(fmt.Sprintf("invalid data type for %s, expect string but found %s", p, t))
+				}
+			case xsql.DATETIME:
+				return errors.New(fmt.Sprintf("invalid data type for %s, datetime type is not supported yet", p))
+			case xsql.BOOLEAN:
+				if jtype == reflect.Bool{
+					r[p] = t.(bool)
+				}else{
+					return errors.New(fmt.Sprintf("invalid data type for %s, expect boolean but found %s", p, t))
+				}
+			default:
+				return errors.New(fmt.Sprintf("invalid data type for %s, it is not supported yet", st))
+			}
+		case *xsql.ArrayType:
+			if jtype != reflect.Slice{
+				return errors.New(fmt.Sprintf("invalid data type for %s, expect array but found %s", p, t))
+			}
+			if tempArr, err := addArrayField(st, t.([]interface{})); err !=nil{
+				return err
+			}else {
+				r[p] = tempArr
+			}
+		case *xsql.RecType:
+			if jtype != reflect.Map{
+				return errors.New(fmt.Sprintf("invalid data type for %s, expect struct but found %s", p, t))
+			}
+			nextJ, ok := j[p].(map[string]interface{})
+			if !ok {
+				return errors.New(fmt.Sprintf("invalid data type for %s, expect map but found %s", p, t))
+			}
+			nextR := make(map[string]interface{})
+			for _, nextF := range st.StreamFields {
+				nextP := nextF.Name
+				if e := addRecField(nextF.FieldType, nextR, nextJ, nextP); e != nil{
+					return e
+				}
+			}
+			r[p] = nextR
+		default:
+			return errors.New(fmt.Sprintf("unsupported type %T", st))
+		}
+		return nil
+	}else{
+		return errors.New(fmt.Sprintf("invalid data %s, field %s not found", j, p))
+	}
+}
+
+//ft must be xsql.ArrayType
+//side effect: r[p] will be set to the new array
+func addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, error) {
+	if ft.FieldType != nil { //complex type array or struct
+		switch st := ft.FieldType.(type) { //Only two complex types supported here
+		case *xsql.ArrayType: //TODO handle array of array. Now the type is treated as interface{}
+			var tempSlice [][]interface{}
+			for i, t := range srcSlice{
+				if reflect.ValueOf(t).Kind() == reflect.Array{
+					if tempArr, err := addArrayField(st, t.([]interface{})); err !=nil{
+						return nil, err
+					}else {
+						tempSlice = append(tempSlice, tempArr.([]interface{}))
+					}
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect array but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		case *xsql.RecType:
+			var tempSlice []map[string]interface{}
+			for i, t := range srcSlice{
+				if reflect.ValueOf(t).Kind() == reflect.Map{
+					j, ok := t.(map[string]interface{})
+					if !ok {
+						return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect map but found %s", i, t))
+					}
+					r := make(map[string]interface{})
+					for _, f := range st.StreamFields {
+						p := f.Name
+						if e := addRecField(f.FieldType, r, j, p); e != nil{
+							return nil, e
+						}
+					}
+					tempSlice = append(tempSlice, r)
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		default:
+			return nil, errors.New(fmt.Sprintf("unsupported type %T", st))
+		}
+	}else{ //basic type
+		switch ft.Type {
+		case xsql.UNKNOWN:
+			return nil, errors.New(fmt.Sprintf("invalid data type unknown defined for %s, please checke the stream definition", srcSlice))
+		case xsql.BIGINT:
+			var tempSlice []int
+			for i, t := range srcSlice {
+				if reflect.ValueOf(t).Kind() == reflect.Float64{
+					tempSlice = append(tempSlice, int(t.(float64)))
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		case xsql.FLOAT:
+			var tempSlice []float64
+			for i, t := range srcSlice {
+				if reflect.ValueOf(t).Kind() == reflect.Float64{
+					tempSlice = append(tempSlice, t.(float64))
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		case xsql.STRINGS:
+			var tempSlice []string
+			for i, t := range srcSlice {
+				if reflect.ValueOf(t).Kind() == reflect.String{
+					tempSlice = append(tempSlice, t.(string))
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect string but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		case xsql.DATETIME:
+			return nil, errors.New(fmt.Sprintf("invalid data type for %s, datetime type is not supported yet", srcSlice))
+		case xsql.BOOLEAN:
+			var tempSlice []bool
+			for i, t := range srcSlice {
+				if reflect.ValueOf(t).Kind() == reflect.Bool{
+					tempSlice = append(tempSlice, t.(bool))
+				}else{
+					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect boolean but found %s", i, t))
+				}
+			}
+			return tempSlice, nil
+		default:
+			return nil, errors.New(fmt.Sprintf("invalid data type for %T, datetime type is not supported yet", ft.Type))
+		}
+	}
+}

xsql/plans/preprocessor_test.go

@@ -0,0 +1,162 @@
+package plans
+
+import (
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestPreprocessor_Apply(t *testing.T) {
+
+	var tests = []struct {
+		stmt *xsql.StreamStmt
+		data []byte
+		result interface{}
+	}{
+		//Basic type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.BIGINT}},
+				},
+			},
+			data: []byte(`{"a": 6}`),
+			result: nil,
+		},
+
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.BIGINT}},
+				},
+			},
+			data: []byte(`{"abc": 6}`),
+			result: map[string]interface{}{
+				"abc" : int(6),
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+				},
+			},
+			data: []byte(`{"abc": 34, "def" : "hello", "ghi": 50}`),
+			result: map[string]interface{}{
+				"abc" : float64(34),
+				"def" : "hello",
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.BOOLEAN}},
+				},
+			},
+			data: []byte(`{"abc": 77, "def" : "hello"}`),
+			result: nil,
+		},
+		//Array type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.BOOLEAN}},
+				},
+			},
+			data: []byte(`{"a": {"b" : "hello"}}`),
+			result: nil,
+		},
+		//Rec type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "a", FieldType: &xsql.RecType{
+						StreamFields: []xsql.StreamField{
+							{Name: "b", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+						},
+					}},
+				},
+			},
+			data: []byte(`{"a": {"b" : "hello"}}`),
+			result: map[string]interface{}{
+				"a" : map[string]interface{}{
+					"b": "hello",
+				},
+			},
+		},
+
+		//Array of complex type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "a", FieldType: &xsql.ArrayType{
+						Type: xsql.STRUCT,
+						FieldType: &xsql.RecType{
+							StreamFields: []xsql.StreamField{
+								{Name: "b", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+							},
+						},
+					}},
+				},
+			},
+			data: []byte(`{"a": [{"b" : "hello1"}, {"b" : "hello2"}]}`),
+			result: map[string]interface{}{
+				"a" : []map[string]interface{}{
+					{"b": "hello1"},
+					{"b": "hello2"},
+				},
+			},
+		},
+		//Rec of complex type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "a", FieldType: &xsql.RecType{
+						StreamFields: []xsql.StreamField{
+							{Name: "b", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+							{Name: "c", FieldType: &xsql.RecType{
+								StreamFields: []xsql.StreamField{
+									{Name: "d", FieldType: &xsql.BasicType{Type: xsql.BIGINT}},
+								},
+							}},
+						},
+					}},
+				},
+			},
+			data: []byte(`{"a": {"b" : "hello", "c": {"d": 35.2}}}`),
+			result: map[string]interface{}{
+				"a" : map[string]interface{}{
+					"b" : "hello",
+					"c" : map[string]interface{}{
+						"d": int(35),
+					},
+				},
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+
+	defer common.CloseLogger()
+	for i, tt := range tests {
+
+		pp := &Preprocessor{StreamStmt:tt.stmt}
+		result := pp.Apply(nil, tt.data)
+		if !reflect.DeepEqual(tt.result, result) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.data, tt.result, result)
+		}
+	}
+}

xsql/plans/project_operator.go

@@ -0,0 +1,66 @@
+package plans
+
+import (
+	"context"
+	"encoding/json"
+	"engine/xsql"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type ProjectPlan struct {
+	Fields xsql.Fields
+}
+
+func (pp *ProjectPlan) Apply(ctx context.Context, data interface{}) interface{} {
+	var input map[string]interface{}
+	if d, ok := data.(map[string]interface{}); !ok {
+		fmt.Printf("Expect map[string]interface{} type.\n")
+		return nil
+	} else {
+		input = d
+	}
+
+	var result = make(map[string]interface{})
+
+	ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(xsql.MapValuer(input), &xsql.FunctionValuer{}, &xsql.WildcardValuer{Data: input})}
+
+	for _, f := range pp.Fields {
+		v := ve.Eval(f.Expr)
+		if val, ok := v.(map[string]interface{}); ok { //It should be the asterisk in fields list.
+			result = val
+			break
+		} else {
+			result[assignName(f.Name, f.AName, result)] = v
+		}
+	}
+
+	if ret, err := json.Marshal(result); err == nil {
+		return ret
+	} else {
+		fmt.Printf("Found error: %v.\n", err)
+		return nil
+	}
+}
+
+const DEFAULT_FIELD_NAME_PREFIX string = "rengine_field_"
+
+func assignName(name, alias string, fields map[string] interface{}) string {
+	if result := strings.Trim(alias, " "); result != "" {
+		return result
+	}
+
+	if result := strings.Trim(name, " "); result != "" {
+		return result
+	}
+
+	for i := 0; i < 2048; i++ {
+		key := DEFAULT_FIELD_NAME_PREFIX + strconv.Itoa(i)
+		if _, ok := fields[key]; !ok {
+			return key
+		}
+	}
+	fmt.Printf("Cannot assign a default field name.\n")
+	return ""
+}

xsql/plans/project_test.go

@@ -0,0 +1,183 @@
+package plans
+
+import (
+	"encoding/json"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestProjectPlan_Apply1(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data string
+		result map[string]interface{}
+	}{
+		{
+			sql: "SELECT a FROM test",
+			data: `{"a": "val_a"}`,
+			result: map[string]interface{}{
+				"a" : "val_a",
+			},
+		},
+
+		{
+			sql: `SELECT "value" FROM test`,
+			data: `{}`,
+			result: map[string]interface{}{
+				DEFAULT_FIELD_NAME_PREFIX + "0" : "value",
+			},
+		},
+
+		{
+			sql: `SELECT 3.4 FROM test`,
+			data: `{}`,
+			result: map[string]interface{}{
+				DEFAULT_FIELD_NAME_PREFIX + "0" : 3.4,
+			},
+		},
+
+		{
+			sql: `SELECT 5 FROM test`,
+			data: `{}`,
+			result: map[string]interface{}{
+				DEFAULT_FIELD_NAME_PREFIX + "0" : 5.0,
+			},
+		},
+
+		{
+			sql: `SELECT a, "value" AS b FROM test`,
+			data: `{"a": "val_a"}`,
+			result: map[string]interface{}{
+				"a" : "val_a",
+				"b" : "value",
+			},
+		},
+
+		{
+			sql: `SELECT a, "value" AS b, 3.14 as Pi, 0 as Zero FROM test`,
+			data: `{"a": "val_a"}`,
+			result: map[string]interface{}{
+				"a" : "val_a",
+				"b" : "value",
+				"Pi" : 3.14,
+				"Zero" : 0.0,
+			},
+		},
+
+		{
+			sql: `SELECT a->b AS ab FROM test`,
+			data: `{"a": {"b" : "hello"}}`,
+			result: map[string]interface{}{
+				"ab" : "hello",
+			},
+		},
+
+		{
+			sql: `SELECT a[0]->b AS ab FROM test`,
+			data: `{"a": [{"b" : "hello1"}, {"b" : "hello2"}]}`,
+			result: map[string]interface{}{
+				"ab" : "hello1",
+			},
+		},
+
+		{
+			sql: `SELECT a->c->d AS f1 FROM test`,
+			data: `{"a": {"b" : "hello", "c": {"d": 35.2}}}`,
+			result: map[string]interface{}{
+				"f1" : 35.2,
+			},
+		},
+
+		//The int type is not supported yet, the json parser returns float64 for int values
+		{
+			sql: `SELECT a->c->d AS f1 FROM test`,
+			data: `{"a": {"b" : "hello", "c": {"d": 35}}}`,
+			result: map[string]interface{}{
+				"f1" : float64(35),
+			},
+		},
+
+		{
+			sql: "SELECT a FROM test",
+			data: `{}`,
+			result: map[string]interface{}{
+			},
+		},
+
+		{
+			sql: "SELECT * FROM test",
+			data: `{}`,
+			result: map[string]interface{}{
+			},
+		},
+
+		{
+			sql: `SELECT * FROM test`,
+			data: `{"a": {"b" : "hello", "c": {"d": 35.2}}}`,
+			result: map[string]interface{}{
+				"a" : map[string]interface{} {
+					"b" : "hello",
+					"c" : map[string]interface{} {
+						"d" : 35.2,
+					},
+				},
+			},
+		},
+
+		{
+			sql: `SELECT * FROM test`,
+			data: `{"a": "val1", "b": 3.14}`,
+			result: map[string]interface{}{
+				"a" : "val1",
+				"b" : 3.14,
+			},
+		},
+
+		{
+			sql: `SELECT 3*4 AS f1 FROM test`,
+			data: `{}`,
+			result: map[string]interface{}{
+				"f1" : float64(12),
+			},
+		},
+
+		{
+			sql: `SELECT 4.5*2 AS f1 FROM test`,
+			data: `{}`,
+			result: map[string]interface{}{
+				"f1" : float64(9),
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, _ := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+
+		pp := &ProjectPlan{Fields:stmt.Fields}
+		var input map[string]interface{}
+		if err := json.Unmarshal([]byte(tt.data), &input); err != nil {
+			fmt.Printf("Failed to parse the JSON data.\n")
+			return
+		}
+		result := pp.Apply(nil, input)
+		var mapRes map[string]interface{}
+		if v, ok := result.([]byte); ok {
+			err := json.Unmarshal(v, &mapRes)
+			if err != nil {
+				t.Errorf("Failed to parse the input into map.\n")
+				continue
+			}
+			//fmt.Printf("%t\n", mapRes["rengine_field_0"])
+
+			if !reflect.DeepEqual(tt.result, mapRes) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, mapRes)
+			}
+		} else {
+			t.Errorf("The returned result is not type of []byte\n")
+		}
+	}
+}

xsql/processors/xsql_processor.go

@@ -0,0 +1,253 @@
+package processors
+
+import (
+	"bytes"
+	"engine/common"
+	"engine/xsql"
+	"engine/xsql/plans"
+	"engine/xstream"
+	"engine/xstream/collectors"
+	"engine/xstream/extensions"
+	"fmt"
+	"github.com/dgraph-io/badger"
+	"strings"
+)
+
+var log = common.Log
+
+type StreamProcessor struct {
+	statement string
+	badgerDir string
+}
+
+//@params s : the sql string of create stream statement
+//@params d : the directory of the badger DB to save the stream info
+func NewStreamProcessor(s, d string) *StreamProcessor {
+	processor := &StreamProcessor{
+		statement: s,
+		badgerDir: d,
+	}
+	return processor
+}
+
+
+func (p *StreamProcessor) Exec() (result []string, err error) {
+	parser := xsql.NewParser(strings.NewReader(p.statement))
+	stmt, err := xsql.Language.Parse(parser)
+	if err != nil {
+		return
+	}
+
+	db, err := common.DbOpen(p.badgerDir)
+	if err != nil {
+		return
+	}
+	defer common.DbClose(db)
+
+	switch s := stmt.(type) {
+	case *xsql.StreamStmt:
+		var r string
+		r, err = p.execCreateStream(s, db)
+		result = append(result, r)
+	case *xsql.ShowStreamsStatement:
+		result, err = p.execShowStream(s, db)
+	case *xsql.DescribeStreamStatement:
+		var r string
+		r, err = p.execDescribeStream(s, db)
+		result = append(result, r)
+	case *xsql.ExplainStreamStatement:
+		var r string
+		r, err = p.execExplainStream(s, db)
+		result = append(result, r)
+	case *xsql.DropStreamStatement:
+		var r string
+		r, err = p.execDropStream(s, db)
+		result = append(result, r)
+	}
+
+	return
+}
+
+func (p *StreamProcessor) execCreateStream(stmt *xsql.StreamStmt, db *badger.DB) (string, error) {
+	err := common.DbSet(db, string(stmt.Name), p.statement)
+	if err != nil {
+		return "", err
+	}else{
+		return fmt.Sprintf("stream %s created", stmt.Name), nil
+	}
+}
+
+func (p *StreamProcessor) execShowStream(stmt *xsql.ShowStreamsStatement, db *badger.DB) ([]string,error) {
+	keys, err := common.DbKeys(db)
+	if len(keys) == 0 {
+		keys = append(keys, "no stream definition found")
+	}
+	return keys, err
+}
+
+func (p *StreamProcessor) execDescribeStream(stmt *xsql.DescribeStreamStatement, db *badger.DB) (string,error) {
+	s, err := common.DbGet(db, string(stmt.Name))
+	if err != nil {
+		return "", fmt.Errorf("stream %s not found", stmt.Name)
+	}
+
+	parser := xsql.NewParser(strings.NewReader(s))
+	stream, err := xsql.Language.Parse(parser)
+	streamStmt, ok := stream.(*xsql.StreamStmt)
+	if !ok{
+		return "", fmt.Errorf("error resolving the stream %s, the data in db may be corrupted", stmt.Name)
+	}
+	var buff bytes.Buffer
+	buff.WriteString("Fields\n--------------------------------------------------------------------------------\n")
+	for _, f := range streamStmt.StreamFields {
+		buff.WriteString(f.Name + "\t")
+		xsql.PrintFieldType(f.FieldType, &buff)
+		buff.WriteString("\n")
+	}
+	buff.WriteString("\n")
+	common.PrintMap(streamStmt.Options, &buff)
+	return buff.String(), err
+}
+
+func (p *StreamProcessor) execExplainStream(stmt *xsql.ExplainStreamStatement, db *badger.DB) (string,error) {
+	_, err := common.DbGet(db, string(stmt.Name))
+	if err != nil{
+		return "", fmt.Errorf("stream %s not found", stmt.Name)
+	}
+	return "TO BE SUPPORTED", nil
+}
+
+func (p *StreamProcessor) execDropStream(stmt *xsql.DropStreamStatement, db *badger.DB) (string, error) {
+	err := common.DbDelete(db, string(stmt.Name))
+	if err != nil {
+		return "", err
+	}else{
+		return fmt.Sprintf("stream %s dropped", stmt.Name), nil
+	}
+}
+
+func GetStream(db *badger.DB, name string) (stmt *xsql.StreamStmt, err error){
+	s, err := common.DbGet(db, string(name))
+	if err != nil {
+		return
+	}
+
+	parser := xsql.NewParser(strings.NewReader(s))
+	stream, err := xsql.Language.Parse(parser)
+	stmt, ok := stream.(*xsql.StreamStmt)
+	if !ok{
+		err = fmt.Errorf("error resolving the stream %s, the data in db may be corrupted", name)
+	}
+	return
+}
+
+
+type RuleProcessor struct {
+	sql string
+//	actions string
+	badgerDir string
+}
+
+func NewRuleProcessor(s, d string) *RuleProcessor {
+	processor := &RuleProcessor{
+		sql: s,
+		badgerDir: d,
+	}
+	return processor
+}
+
+func (p *RuleProcessor) Exec() error {
+	parser := xsql.NewParser(strings.NewReader(p.sql))
+	if stmt, err := xsql.Language.Parse(parser); err != nil{
+		return fmt.Errorf("parse sql %s error: %s", p.sql , err)
+	}else{
+		if selectStmt, ok := stmt.(*xsql.SelectStatement); !ok{
+			return fmt.Errorf("sql %s is not a select statement", p.sql)
+		}else{
+			//TODO Validation here or in the cli?
+			tp := xstream.New()
+
+			//create sources and preprocessor
+			db, err := common.DbOpen(p.badgerDir)
+			if err != nil {
+				return err
+			}
+			defer common.DbClose(db)
+			var inputs []xstream.Emitter
+			for _, s := range selectStmt.Sources {
+				switch t := s.(type){
+				case *xsql.Table:
+					if streamStmt, err := GetStream(db, t.Name); err != nil{
+						return err
+					} else {
+						mqs, err := extensions.NewWithName(string(streamStmt.Name), streamStmt.Options["DATASOURCE"], streamStmt.Options["CONF_KEY"])
+						if err != nil {
+							return err
+						}
+						tp.AddSrc(mqs)
+
+						preprocessorOp := xstream.Transform(&plans.Preprocessor{StreamStmt: streamStmt}, "preprocessor_" +t.Name)
+						tp.AddOperator([]xstream.Emitter{mqs}, preprocessorOp)
+						inputs = append(inputs, preprocessorOp)
+					}
+				default:
+					return fmt.Errorf("unsupported source type %T", s)
+				}
+			}
+
+			//if selectStmt.Joins != nil {
+			//	for _, join := range selectStmt.Joins {
+			//		if streamStmt, err := GetStream(db, join.Name); err != nil{
+			//			return err
+			//		} else {
+			//			mqs, err := extensions.NewWithName(string(streamStmt.Name), streamStmt.Options["DATASOURCE"], streamStmt.Options["CONF_KEY"])
+			//			if err != nil {
+			//				return err
+			//			}
+			//			tp.AddSrc(mqs)
+			//
+			//			preprocessorOp := xstream.Transform(&plans.Preprocessor{StreamStmt: streamStmt}, "preprocessor_" + join.Name)
+			//			tp.AddOperator([]xstream.Emitter{mqs}, preprocessorOp)
+			//			inputs = append(inputs, preprocessorOp)
+			//		}
+			//	}
+			//
+			//	joinOp := xstream.Transform(&plans.JoinPlan{Joins:selectStmt.Joins, Dimensions: selectStmt.Dimensions}, "join")
+			//	//TODO concurrency setting by command
+			//	//joinOp.SetConcurrency(3)
+			//	//TODO Read the ticker from dimension statement
+			//	joinOp.SetTicker(time.Second * 5)
+			//	tp.AddOperator(inputs, joinOp)
+			//	inputs = []xstream.Emitter{joinOp}
+			//}
+
+
+			if selectStmt.Condition != nil {
+				filterOp := xstream.Transform(&plans.FilterPlan{Condition: selectStmt.Condition}, "filter")
+				//TODO concurrency setting by command
+				// filterOp.SetConcurrency(3)
+				tp.AddOperator(inputs, filterOp)
+				inputs = []xstream.Emitter{filterOp}
+			}
+
+			if selectStmt.Fields != nil {
+				projectOp := xstream.Transform(&plans.ProjectPlan{Fields: selectStmt.Fields}, "project")
+				tp.AddOperator(inputs, projectOp)
+				inputs = []xstream.Emitter{projectOp}
+			}
+
+
+			//TODO hard coded sink now. parameterize it
+			tp.AddSink(inputs, collectors.Func(func(data interface{}) error {
+				fmt.Printf("Sink: %s\n", data)
+				return nil
+			}))
+
+			if err := <-tp.Open(); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+

xsql/processors/xsql_processor_test.go

@@ -0,0 +1,88 @@
+package processors
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestStreamCreateProcessor(t *testing.T) {
+	const BadgerDir = "D:\\tmp\\test"
+	var tests = []struct {
+		s    string
+		r    []string
+		err  string
+	}{
+		{
+			s: `SHOW STREAMS;`,
+			r: []string{"no stream definition found"},
+		},
+		{
+			s: `EXPLAIN STREAM demo;`,
+			err: "stream demo not found",
+		},
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+					FIRST_NAME STRING,
+					LAST_NAME STRING,
+					NICKNAMES ARRAY(STRING),
+					Gender BOOLEAN,
+					ADDRESS STRUCT(STREET_NAME STRING, NUMBER BIGINT),
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID");`,
+			r: []string{"stream demo created"},
+		},
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID");`,
+			err: "key demo already exist, delete it before creating a new one",
+		},
+		{
+			s: `EXPLAIN STREAM demo;`,
+			r: []string{"TO BE SUPPORTED"},
+		},
+		{
+			s: `DESCRIBE STREAM demo;`,
+			r: []string{"Fields\n--------------------------------------------------------------------------------\nUSERID\tbigint\nFIRST_NAME\tstring\nLAST_NAME\tstring\nNICKNAMES\t" +
+				"array(string)\nGender\tboolean\nADDRESS\tstruct(STREET_NAME string, NUMBER bigint)\n\n" +
+				"DATASOURCE: users\nFORMAT: AVRO\nKEY: USERID\n"},
+		},
+		{
+			s: `SHOW STREAMS;`,
+			r: []string{"demo"},
+		},
+		{
+			s: `DROP STREAM demo;`,
+			r: []string{"stream demo dropped"},
+		},
+		{
+			s: `DESCRIBE STREAM demo;`,
+			err: "stream demo not found",
+		},
+		{
+			s: `DROP STREAM demo;`,
+			err: "Key not found",
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+
+	for i, tt := range tests {
+		results, err := NewStreamProcessor(tt.s, BadgerDir).Exec()
+		if !reflect.DeepEqual(tt.err, errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
+		} else if tt.err == "" {
+			if !reflect.DeepEqual(tt.r, results) {
+				t.Errorf("%d. %q\n\nstmt mismatch:\n\ngot=%#v\n\n", i, tt.s, results)
+			}
+		}
+	}
+}
+func errstring(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}
+

xsql/util.go

@@ -0,0 +1,31 @@
+package xsql
+
+import "bytes"
+
+func PrintFieldType(ft FieldType, buff *bytes.Buffer) {
+	switch t := ft.(type) {
+	case *BasicType:
+		buff.WriteString(t.Type.String())
+	case *ArrayType:
+		buff.WriteString("array(")
+		if t.FieldType != nil {
+			PrintFieldType(t.FieldType, buff)
+		}else{
+			buff.WriteString(t.Type.String())
+		}
+		buff.WriteString(")")
+	case *RecType:
+		buff.WriteString("struct(")
+		isFirst := true
+		for _, f := range t.StreamFields {
+			if isFirst{
+				isFirst = false
+			}else{
+				buff.WriteString(", ")
+			}
+			buff.WriteString(f.Name + " ")
+			PrintFieldType(f.FieldType, buff)
+		}
+		buff.WriteString(")")
+	}
+}

xsql/visitor.go

@@ -0,0 +1,40 @@
+package xsql
+
+type ExpressionVisitor interface {
+	Visit(Node) Visitor
+
+	VisitBinaryExpr(*BinaryExpr)
+	VisitFieldRef(*FieldRef)
+	VisitIntegerLiteral(*IntegerLiteral)
+}
+
+type ExpressionVisitorAdaptor struct {
+}
+
+func (eva *ExpressionVisitorAdaptor) DoVisit(v ExpressionVisitor, expr Node) {
+	switch n := expr.(type) {
+	case *BinaryExpr:
+		v.VisitBinaryExpr(n)
+	case *FieldRef:
+		v.VisitFieldRef(n)
+	case *IntegerLiteral:
+		v.VisitIntegerLiteral(n)
+	}
+}
+
+func (eva *ExpressionVisitorAdaptor) Visit(expr Node) Visitor {
+	return nil
+}
+
+func (eva *ExpressionVisitorAdaptor) VisitBinaryExpr(expr *BinaryExpr) {
+	Walk(eva, expr.LHS)
+	Walk(eva, expr.RHS)
+}
+
+func (eva *ExpressionVisitorAdaptor) VisitFieldRef(expr *FieldRef) {
+	Walk(eva, expr)
+}
+
+func (eva *ExpressionVisitorAdaptor) visitIntegerLiteral(expr *FieldRef) {
+
+}

xsql/xsql_manager.go

@@ -0,0 +1,64 @@
+package xsql
+
+import "fmt"
+
+var Language = &ParseTree{}
+
+type ParseTree struct {
+	Handlers map[Token]func(*Parser) (Statement, error)
+	Tokens   map[Token]*ParseTree
+	Keys     []string
+}
+
+func (t *ParseTree) Handle(tok Token, fn func(*Parser) (Statement, error)) {
+	// Verify that there is no conflict for this token in this parse tree.
+	if _, conflict := t.Tokens[tok]; conflict {
+		panic(fmt.Sprintf("conflict for token %s", tok))
+	}
+
+	if _, conflict := t.Handlers[tok]; conflict {
+		panic(fmt.Sprintf("conflict for token %s", tok))
+	}
+
+	if t.Handlers == nil {
+		t.Handlers = make(map[Token]func(*Parser) (Statement, error))
+	}
+	t.Handlers[tok] = fn
+	t.Keys = append(t.Keys, tok.String())
+}
+
+
+func (pt *ParseTree) Parse(p *Parser) (Statement, error) {
+	tok, _ := p.scanIgnoreWhitespace();
+	p.unscan()
+	if f, ok  := pt.Handlers[tok]; ok {
+		return f(p)
+	}
+	return nil, nil
+}
+
+func init() {
+	Language.Handle(SELECT, func(p *Parser) (Statement, error) {
+		return p.Parse()
+	})
+
+	Language.Handle(CREATE, func(p *Parser) (statement Statement, e error) {
+		return p.ParseCreateStreamStmt()
+	})
+
+	Language.Handle(SHOW, func(p *Parser) (statement Statement, e error) {
+		return p.parseShowStreamsStmt()
+	})
+
+	Language.Handle(EXPLAIN, func(p *Parser) (statement Statement, e error) {
+		return p.parseExplainStreamsStmt()
+	})
+
+	Language.Handle(DESCRIBE, func(p *Parser) (statement Statement, e error) {
+		return p.parseDescribeStreamStmt()
+	})
+
+	Language.Handle(DROP, func(p *Parser) (statement Statement, e error) {
+		return p.parseDropStreamsStmt()
+	})
+}

xsql/xsql_parser_tree_test.go

@@ -0,0 +1,87 @@
+package xsql
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParser_ParseTree(t *testing.T) {
+	var tests = []struct {
+		s    string
+		stmt Statement
+		err  string
+	}{
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					StreamField{Name: "USERID", FieldType: &BasicType{Type: BIGINT}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "JSON",
+					"KEY" : "USERID",
+				},
+			},
+		},
+
+		{
+			s: `SHOW STREAMS`,
+			stmt: &ShowStreamsStatement{},
+		},
+
+		{
+			s: `SHOW STREAMSf`,
+			stmt: nil,
+			err: `found "STREAMSf", expected keyword streams.`,
+		},
+
+		{
+			s: `SHOW STREAMS d`,
+			stmt: nil,
+			err: `found "d", expected semecolon or EOF.`,
+		},
+
+		{
+			s: `DESCRIBE STREAM demo`,
+			stmt: &DescribeStreamStatement{
+				Name: "demo",
+			},
+			err: ``,
+		},
+
+		{
+			s: `EXPLAIN STREAM demo1`,
+			stmt: &ExplainStreamStatement{
+				Name: "demo1",
+			},
+			err: ``,
+		},
+
+		{
+			s: `DROP STREAM demo1`,
+			stmt: &DropStreamStatement{
+				Name: "demo1",
+			},
+			err: ``,
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		p := NewParser(strings.NewReader(tt.s))
+		stmt, err := Language.Parse(p)
+		if !reflect.DeepEqual(tt.err, errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
+		} else if tt.err == "" && !reflect.DeepEqual(tt.stmt, stmt) {
+			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.stmt, stmt)
+		}
+	}
+
+}

xsql/xsql_stream_test.go

@@ -0,0 +1,288 @@
+package xsql
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestParser_ParseCreateStream(t *testing.T) {
+	var tests = []struct {
+		s    string
+		stmt *StreamStmt
+		err  string
+	}{
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+					FIRST_NAME STRING,
+					LAST_NAME STRING,
+					NICKNAMES ARRAY(STRING),
+					Gender BOOLEAN,
+					ADDRESS STRUCT(STREET_NAME STRING, NUMBER BIGINT),
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID", CONF_KEY="srv1", type="MQTT");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "USERID", FieldType: &BasicType{Type: BIGINT}},
+					{Name: "FIRST_NAME", FieldType: &BasicType{Type: STRINGS}},
+					{Name: "LAST_NAME", FieldType: &BasicType{Type: STRINGS}},
+					{Name: "NICKNAMES", FieldType: &ArrayType{Type: STRINGS}},
+					{Name: "Gender", FieldType: &BasicType{Type: BOOLEAN}},
+					{Name: "ADDRESS", FieldType: &RecType{
+						StreamFields: []StreamField{
+							{Name: "STREET_NAME", FieldType: &BasicType{Type: STRINGS}},
+							{Name: "NUMBER", FieldType: &BasicType{Type: BIGINT}},
+						},
+					}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+					"CONF_KEY" : "srv1",
+					"TYPE" : "MQTT",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID", STRICT_VALIDATION="true");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "USERID", FieldType: &BasicType{Type: BIGINT}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "JSON",
+					"KEY" : "USERID",
+					"STRICT_VALIDATION" : "true",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+					ADDRESSES ARRAY(STRUCT(STREET_NAME STRING, NUMBER BIGINT)),
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID", STRICT_VALIDATION="FAlse");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "ADDRESSES", FieldType: &ArrayType{
+						Type: STRUCT,
+						FieldType: &RecType{
+							StreamFields: []StreamField{
+								{Name: "STREET_NAME", FieldType: &BasicType{Type: STRINGS}},
+								{Name: "NUMBER", FieldType: &BasicType{Type: BIGINT}},
+							},
+						},
+					}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+					"STRICT_VALIDATION": "FAlse",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+					ADDRESSES ARRAY(STRUCT(STREET_NAME STRING, NUMBER BIGINT)),
+					birthday datetime,
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "ADDRESSES", FieldType: &ArrayType{
+						Type: STRUCT,
+						FieldType: &RecType{
+							StreamFields: []StreamField{
+								{Name: "STREET_NAME", FieldType: &BasicType{Type: STRINGS}},
+								{Name: "NUMBER", FieldType: &BasicType{Type: BIGINT}},
+							},
+						},
+					}},
+					{Name: "birthday", FieldType: &BasicType{Type: DATETIME}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+					NAME string,
+					ADDRESSES ARRAY(STRUCT(STREET_NAME STRING, NUMBER BIGINT)),
+					birthday datetime,
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "NAME", FieldType: &BasicType{Type: STRINGS}},
+					{Name: "ADDRESSES", FieldType: &ArrayType{
+						Type: STRUCT,
+						FieldType: &RecType{
+							StreamFields: []StreamField{
+								{Name: "STREET_NAME", FieldType: &BasicType{Type: STRINGS}},
+								{Name: "NUMBER", FieldType: &BasicType{Type: BIGINT}},
+							},
+						},
+					}},
+					{Name: "birthday", FieldType: &BasicType{Type: DATETIME}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+		
+				) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: nil,
+			err: `found ")", expect stream field name.`,
+		},
+
+		{
+			s: `CREATE STREAM demo (NAME string)
+				 WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID", STRICT_VALIDATION="true1");`, //Invalid STRICT_VALIDATION value
+			stmt: nil,
+			err: `found "true1", expect TRUE/FALSE value in STRICT_VALIDATION option.`,
+		},
+		
+		{
+			s: `CREATE STREAM demo (NAME string) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: []StreamField{
+					{Name: "NAME", FieldType: &BasicType{Type: STRINGS}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "JSON",
+					"KEY" : "USERID",
+				},
+			},
+		},
+		
+		{
+			s: `CREATE STREAM demo (NAME string)) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found ")", expect stream options.`,
+		},
+		
+		{
+			s: `CREATE STREAM demo (NAME string) WITHs (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: StreamName("demo"),
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found "WITHs", expected is with.`,
+		},
+		
+		{
+			s: `CREATE STREAM demo (NAME integer) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: "demo",
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found "integer", expect valid stream field types(BIGINT | FLOAT | STRINGS | DATETIME | BOOLEAN | ARRAY | STRUCT).`,
+		},
+		
+		{
+			s: `CREATE STREAM demo (NAME string) WITH (sources="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: "demo",
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found "sources", unknown option keys(DATASOURCE|FORMAT|KEY|CONF_KEY|STRICT_VALIDATION|TYPE).`,
+		},
+		
+		{
+			s: `CREATE STREAM demo ((NAME string) WITH (DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: "demo",
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found "(", expect stream field name.`,
+		},
+		
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH ();`,
+			stmt: &StreamStmt{
+				Name: "demo",
+				StreamFields: []StreamField{
+					{Name: "USERID", FieldType: &BasicType{Type: BIGINT}},
+				},
+				Options: map[string]string{},
+			},
+		},
+
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH ());`,
+			stmt: &StreamStmt{
+				Name: "",
+				StreamFields: nil,
+				Options: nil,
+			},
+			err: `found ")", expected semicolon or EOF.`,
+		},
+
+		{
+			s: `CREATE STREAM demo (
+					USERID BIGINT,
+				) WITH DATASOURCE="users", FORMAT="JSON", KEY="USERID");`,
+			stmt: &StreamStmt{
+				Name: "",
+				StreamFields: nil,
+				Options: nil,
+			},
+			//TODO The error string should be more accurate
+			err: `found "DATASOURCE", expect stream options.`,
+
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := NewParser(strings.NewReader(tt.s)).ParseCreateStreamStmt()
+		if !reflect.DeepEqual(tt.err, errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
+		} else if tt.err == "" && !reflect.DeepEqual(tt.stmt, stmt) {
+			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.stmt, stmt)
+		}
+	}
+
+}
+
+// errstring returns the string representation of an error.
+//func errstring(err error) string {
+//	if err != nil {
+//		return err.Error()
+//	}
+//	return ""
+//}

xstream/cli/main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"bufio"
+	"engine/common"
+	"engine/xsql/processors"
+	"fmt"
+	"github.com/urfave/cli"
+	"os"
+	"sort"
+	"strings"
+)
+
+var log = common.Log
+
+
+func main() {
+	app := cli.NewApp()
+	app.Version = "0.1"
+
+	//nflag := []cli.Flag { cli.StringFlag{
+	//		Name: "name, n",
+	//		Usage: "the name of stream",
+	//	}}
+
+	dataDir, err := common.GetDataLoc()
+	if err != nil {
+		log.Panic(err)
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:      "stream",
+			Aliases:   []string{"s"},
+			Usage:     "manage streams",
+			Action:    func(c *cli.Context) error {
+				reader := bufio.NewReader(os.Stdin)
+				var inputs []string
+				for {
+					fmt.Print("xstream > ")
+
+					text, _ := reader.ReadString('\n')
+					inputs = append(inputs, text)
+					// convert CRLF to LF
+					text = strings.Replace(text, "\n", "", -1)
+
+					if strings.ToLower(text) == "quit" || strings.ToLower(text) == "exit" {
+						break
+					} else {
+						content, err := processors.NewStreamProcessor(text, dataDir).Exec()
+						if err != nil {
+							fmt.Printf("stream command error: %s\n", err)
+						}else{
+							for _, c := range content{
+								fmt.Println(c)
+							}
+						}
+					}
+				}
+				return nil
+			},
+		},
+
+		{
+			Name:      "query",
+			Aliases:   []string{"q"},
+			Usage:     "query against stream",
+			Action:    func(c *cli.Context) error {
+				reader := bufio.NewReader(os.Stdin)
+				var inputs []string
+				for {
+					fmt.Print("xstream > ")
+
+					text, _ := reader.ReadString('\n')
+					inputs = append(inputs, text)
+					// convert CRLF to LF
+					text = strings.Replace(text, "\n", "", -1)
+
+					if strings.ToLower(text) == "quit" || strings.ToLower(text) == "exit" {
+						break
+					} else {
+						fmt.Println(text)
+
+						err = processors.NewRuleProcessor(text, dataDir).Exec()
+						if err != nil {
+							fmt.Printf("create topology error : %s\n", err)
+						}else{
+							fmt.Println("topology running")
+						}
+					}
+				}
+				return nil
+			},
+		},
+	}
+
+
+	app.Name = "xstream"
+	app.Usage = "The command line tool for EMQ X stream."
+
+	app.Action = func(c *cli.Context) error {
+		cli.ShowSubcommandHelp(c)
+		//cli.ShowVersion(c)
+
+		return nil
+	}
+
+	sort.Sort(cli.FlagsByName(app.Flags))
+	sort.Sort(cli.CommandsByName(app.Commands))
+
+	err = app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

xstream/collectors/func.go

@@ -0,0 +1,79 @@
+package collectors
+
+import (
+	"context"
+	"engine/common"
+	"errors"
+)
+
+var log = common.Log
+// CollectorFunc is a function used to colllect
+// incoming stream data. It can be used as a
+// stream sink.
+type CollectorFunc func(interface{}) error
+
+// FuncCollector is a colletor that uses a function
+// to collect data.  The specified function must be
+// of type:
+//   CollectorFunc
+type FuncCollector struct {
+	input chan interface{}
+	//logf  api.LogFunc
+	//errf  api.ErrorFunc
+	f     CollectorFunc
+	name  string
+}
+
+// Func creates a new value *FuncCollector that
+// will use the specified function parameter to
+// collect streaming data.
+func Func(f CollectorFunc) *FuncCollector {
+	return &FuncCollector{f: f, input: make(chan interface{}, 1024)}
+}
+
+func (c *FuncCollector) GetName() string  {
+	return c.name
+}
+
+func (c *FuncCollector) GetInput() (chan<- interface{}, string)  {
+	return c.input, c.name
+}
+
+// Open is the starting point that starts the collector
+func (c *FuncCollector) Open(ctx context.Context) <-chan error {
+	//c.logf = autoctx.GetLogFunc(ctx)
+	//c.errf = autoctx.GetErrFunc(ctx)
+
+	log.Println("Opening func collector")
+	result := make(chan error)
+
+	if c.f == nil {
+		err := errors.New("Func collector missing function")
+		log.Println(err)
+		go func() { result <- err }()
+		return result
+	}
+
+	go func() {
+		defer func() {
+			log.Println("Closing func collector")
+			close(result)
+		}()
+
+		for {
+			select {
+			case item, opened := <-c.input:
+				if !opened {
+					return
+				}
+				if err := c.f(item); err != nil {
+					log.Println(err)
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return result
+}

xstream/demo/test.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"engine/common"
+	"engine/xsql"
+	"engine/xsql/plans"
+	"engine/xstream"
+	"engine/xstream/collectors"
+	"engine/xstream/extensions"
+	"strings"
+)
+
+func main() {
+
+	log := common.Log
+
+	demo1Stream, err := xsql.NewParser(strings.NewReader("CREATE STREAM demo1 (count bigint) WITH (source=\"users\", FORMAT=\"AVRO\", KEY=\"USERID\")")).ParseCreateStreamStmt()
+	demo2Stream, err := xsql.NewParser(strings.NewReader("CREATE STREAM demo2 (abc bigint) WITH (source=\"users\", FORMAT=\"AVRO\", KEY=\"USERID\")")).ParseCreateStreamStmt()
+	stmt, err := xsql.NewParser(strings.NewReader("SELECT count FROM demo1 where demo1.count > 3")).Parse()
+	if err != nil {
+		log.Fatal("Failed to parse SQL for %s. \n", err)
+	}
+
+	tp := xstream.New()
+
+	mqs1, err := extensions.NewWithName("srv1", "demo1", "")
+	if err != nil {
+		log.Fatalf("Found error %s.\n", err)
+		return
+	}
+	tp.AddSrc(mqs1)
+
+	mqs2, err := extensions.NewWithName("srv2", "demo2", "")
+	if err != nil {
+		log.Fatalf("Found error %s.\n", err)
+		return
+	}
+	tp.AddSrc(mqs2)
+
+	preprocessorOp1 := xstream.Transform(&plans.Preprocessor{StreamStmt: demo1Stream}, "preprocessor1")
+	tp.AddOperator([]xstream.Emitter{mqs1}, preprocessorOp1)
+
+	preprocessorOp2 := xstream.Transform(&plans.Preprocessor{StreamStmt: demo2Stream}, "preprocessor2")
+	tp.AddOperator([]xstream.Emitter{mqs2}, preprocessorOp2)
+
+	filterOp := xstream.Transform(&plans.FilterPlan{Condition: stmt.Condition}, "filter plan")
+	filterOp.SetConcurrency(3)
+	tp.AddOperator([]xstream.Emitter{preprocessorOp1, preprocessorOp2}, filterOp)
+
+	projectOp := xstream.Transform(&plans.ProjectPlan{Fields: stmt.Fields}, "project plan")
+	tp.AddOperator([]xstream.Emitter{filterOp}, projectOp)
+
+
+	tp.AddSink([]xstream.Emitter{projectOp}, collectors.Func(func(data interface{}) error {
+		log.Println("sink result %s", data)
+		return nil
+	}))
+
+	if err := <-tp.Open(); err != nil {
+		log.Fatal(err)
+		return
+	}
+}

xstream/demo/testWindow.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"engine/common"
+	"engine/xsql"
+	"engine/xsql/plans"
+	"engine/xstream"
+	"engine/xstream/collectors"
+	"engine/xstream/extensions"
+	"engine/xstream/operators"
+	"strings"
+)
+
+func main() {
+
+	log := common.Log
+
+	demo1Stream, err := xsql.NewParser(strings.NewReader("CREATE STREAM demo (count bigint) WITH (datasource=\"demo\", FORMAT=\"AVRO\", KEY=\"USERID\")")).ParseCreateStreamStmt()
+	//demo2Stream, err := xsql.NewParser(strings.NewReader("CREATE STREAM demo2 (abc bigint) WITH (datasource=\"demo2\", FORMAT=\"AVRO\", KEY=\"USERID\")")).ParseCreateStreamStmt()
+	//stmt, err := xsql.NewParser(strings.NewReader("SELECT count FROM demo1 where demo1.count > 3")).Parse()
+	if err != nil {
+		log.Fatal("Failed to parse SQL for %s. \n", err)
+	}
+
+	tp := xstream.New()
+
+	mqs1, err := extensions.NewWithName("srv1", "demo", "")
+	if err != nil {
+		log.Fatalf("Found error %s.\n", err)
+		return
+	}
+	tp.AddSrc(mqs1)
+
+	//mqs2, err := extensions.NewWithName("srv1", "demo2")
+	//if err != nil {
+	//	log.Fatalf("Found error %s.\n", err)
+	//	return
+	//}
+	//tp.AddSrc(mqs2)
+
+	preprocessorOp1 := xstream.Transform(&plans.Preprocessor{StreamStmt: demo1Stream}, "preprocessor1")
+	tp.AddOperator([]xstream.Emitter{mqs1}, preprocessorOp1)
+
+	//preprocessorOp2 := xstream.Transform(&plans.Preprocessor{StreamStmt: demo2Stream}, "preprocessor2")
+	//tp.AddOperator([]xstream.Emitter{mqs2}, preprocessorOp2)
+
+	//filterOp := xstream.Transform(&plans.FilterPlan{Condition: stmt.Condition}, "filter plan")
+	//filterOp.SetConcurrency(3)
+	//tp.AddOperator([]xstream.Emitter{preprocessorOp1, preprocessorOp2}, filterOp)
+	//
+	//projectOp := xstream.Transform(&plans.ProjectPlan{Fields: stmt.Fields}, "project plan")
+	//tp.AddOperator([]xstream.Emitter{filterOp}, projectOp)
+
+	//windowOp := operators.NewWindowOp("windowOp", &operators.WindowConfig{
+	//	Type: operators.NO_WINDOW,
+	//})
+
+	//windowOp := operators.NewWindowOp("windowOp", &operators.WindowConfig{
+	//	Type: operators.TUMBLING_WINDOW,
+	//	Length: 30000,
+	//})
+
+	//windowOp := operators.NewWindowOp("windowOp", &operators.WindowConfig{
+	//	Type: operators.HOPPING_WINDOW,
+	//	Length: 20000,
+	//	Interval: 10000,
+	//})
+	//
+	//windowOp := operators.NewWindowOp("windowOp", &operators.WindowConfig{
+	//	Type: operators.SLIDING_WINDOW,
+	//	Length: 20000,
+	//})
+
+	windowOp := operators.NewWindowOp("windowOp", &operators.WindowConfig{
+		Type: operators.SESSION_WINDOW,
+		Length: 20000,
+		Interval: 6000,
+	})
+
+	tp.AddOperator([]xstream.Emitter{preprocessorOp1}, windowOp)
+
+
+	tp.AddSink([]xstream.Emitter{windowOp}, collectors.Func(func(data interface{}) error {
+		log.Println("sink result %s", data)
+		return nil
+	}))
+
+	if err := <-tp.Open(); err != nil {
+		log.Fatal(err)
+		return
+	}
+}

xstream/extensions/mqtt_source.go

@@ -0,0 +1,161 @@
+package extensions
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	MQTT "github.com/eclipse/paho.mqtt.golang"
+	"github.com/go-yaml/yaml"
+	"github.com/google/uuid"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+var log = common.Log
+type MQTTSource struct {
+	srv      string
+	tpc      string
+	clientid string
+	schema   map[string]interface{}
+
+	outs  map[string]chan<- interface{}
+	conn MQTT.Client
+	name 		string
+	//ctx context.Context
+}
+
+
+type MQTTConfig struct {
+	Qos string `yaml:"qos"`
+	Sharedsubscription string `yaml:"sharedsubscription"`
+	Servers []string `yaml:"servers"`
+	Clientid string `yaml:"clientid"`
+}
+
+
+const confName string = "mqtt_source.yaml"
+
+func NewWithName(name string, topic string, confKey string) (*MQTTSource, error) {
+	confDir, err := common.GetConfLoc()
+	var file string = confDir + confName
+	if err != nil {
+		//Try the development mode, read from workspace
+		file = "xstream/extensions/" + confName
+		if abs, err1 := filepath.Abs(file); err1 != nil {
+			return nil, err1
+		} else {
+			file = abs
+		}
+	}
+
+	b, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var cfg map[string]MQTTConfig
+	if err := yaml.Unmarshal(b, &cfg); err != nil {
+		log.Fatal(err)
+	}
+
+	ms := &MQTTSource{tpc: topic, name: name}
+	ms.outs = make(map[string]chan<- interface{})
+	if srvs := cfg[confKey].Servers; srvs != nil && len(srvs) > 1 {
+		return nil, fmt.Errorf("It only support one server in %s section.", confKey)
+	} else if srvs == nil {
+		srvs = cfg["default"].Servers
+		if srvs != nil && len(srvs) == 1 {
+			ms.srv = srvs[0]
+		} else {
+			return nil, fmt.Errorf("Wrong configuration in default section!")
+		}
+	} else {
+		ms.srv = srvs[0]
+	}
+
+	if cid := cfg[confKey].Clientid; cid != "" {
+		ms.clientid = cid
+	} else {
+		ms.clientid = cfg["default"].Clientid
+	}
+	return ms, nil
+}
+
+func fileExists(filename string) bool {
+	info, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return !info.IsDir()
+}
+
+func (ms *MQTTSource) WithSchema(schema string) *MQTTSource {
+	return ms
+}
+
+func (ms *MQTTSource) GetName() string {
+	return ms.name
+}
+
+func (ms *MQTTSource) AddOutput(output chan<- interface{}, name string) {
+	if _, ok := ms.outs[name]; !ok{
+		ms.outs[name] = output
+	}else{
+		log.Error("fail to add output %s, operator %s already has an output of the same name", name, ms.name)
+	}
+}
+
+func (ms *MQTTSource) Open(ctx context.Context) error {
+	go func() {
+		exeCtx, cancel := context.WithCancel(ctx)
+		opts := MQTT.NewClientOptions().AddBroker(ms.srv)
+
+		if ms.clientid == "" {
+			if uuid, err := uuid.NewUUID(); err != nil {
+				log.Printf("Failed to get uuid, the error is %s.\n", err)
+				cancel()
+				return
+			} else {
+				opts.SetClientID(uuid.String())
+			}
+		} else {
+			opts.SetClientID(ms.clientid)
+		}
+
+		h := func(client MQTT.Client, msg MQTT.Message) {
+			if ms.tpc != msg.Topic() {
+				select {
+				case <-exeCtx.Done():
+					log.Println("Done 1.")
+					ms.conn.Disconnect(5000)
+				}
+				return
+			} else {
+				log.Infof("received %s", msg.Payload())
+				tuple := &xsql.Tuple{EmitterName:ms.name, Message:msg.Payload(), Timestamp: common.TimeToUnixMilli(time.Now())}
+				for _, out := range ms.outs{
+					out <- tuple
+				}
+			}
+		}
+
+		opts.SetDefaultPublishHandler(h)
+		c := MQTT.NewClient(opts)
+		if token := c.Connect(); token.Wait() && token.Error() != nil {
+			log.Fatalf("Found error: %s.\n", token.Error())
+			cancel()
+		}
+		log.Printf("The connection to server %s was established successfully.\n", ms.srv)
+		ms.conn = c
+		if token := c.Subscribe(ms.tpc, 0, nil); token.Wait() && token.Error() != nil {
+			log.Fatalf("Found error: %s.\n", token.Error())
+			cancel()
+		}
+		log.Printf("Successfully subscribe to topic %s.\n", ms.tpc)
+	}()
+
+	return nil
+}

xstream/extensions/mqtt_source.yaml

@@ -0,0 +1,13 @@
+#Global MQTT configurations
+default:
+  qos: 1
+  sharedsubscription: true
+  servers: [tcp://127.0.0.1:1883]
+  clientid: xstream_client
+  #TODO: Other global configurations
+
+
+#Override the global configurations
+demo_conf: #Conf_key
+  qos: 0
+  servers: [tls://10.211.55.6:1883, tcp://127.0.0.1]

xstream/funcs.go

@@ -0,0 +1,158 @@
+package xstream
+
+import (
+	"context"
+	"fmt"
+	"engine/xstream/operators"
+	"reflect"
+
+)
+
+type unaryFuncForm byte
+
+const (
+	unaryFuncUnsupported unaryFuncForm = iota
+	unaryFuncForm1
+	unaryFuncForm2
+)
+
+// ProcessFunc returns a unary function which applies the specified
+// user-defined function that processes data items from upstream and
+// returns a result value. The provided function must be of type:
+//   func(T) R
+//   where T is the type of incoming item
+//   R the type of returned processed item
+func ProcessFunc(f interface{}) (operators.UnFunc, error) {
+	fntype := reflect.TypeOf(f)
+
+	funcForm, err := isUnaryFuncForm(fntype)
+	if err != nil {
+		return nil, err
+	}
+	if funcForm == unaryFuncUnsupported {
+		return nil, fmt.Errorf("unsupported unary func type")
+	}
+
+	fnval := reflect.ValueOf(f)
+
+	return operators.UnFunc(func(ctx context.Context, data interface{}) interface{} {
+		result := callOpFunc(fnval, ctx, data, funcForm)
+		return result.Interface()
+	}), nil
+}
+
+// FilterFunc returns a unary function (api.UnFunc) which applies the user-defined
+// filtering to apply predicates that filters out data items from being included
+// in the downstream.  The provided user-defined function must be of type:
+//   func(T)bool - where T is the type of incoming data item, bool is the value of the predicate
+// When the user-defined function returns false, the current processed data item will not
+// be placed in the downstream processing.
+func FilterFunc(f interface{}) (operators.UnFunc, error) {
+	fntype := reflect.TypeOf(f)
+
+	funcForm, err := isUnaryFuncForm(fntype)
+	if err != nil {
+		return nil, err
+	}
+	if funcForm == unaryFuncUnsupported {
+		return nil, fmt.Errorf("unsupported unary func type")
+	}
+
+	// ensure bool ret type
+	if fntype.Out(0).Kind() != reflect.Bool {
+		return nil, fmt.Errorf("unary filter func must return bool")
+	}
+
+	fnval := reflect.ValueOf(f)
+	return operators.UnFunc(func(ctx context.Context, data interface{}) interface{} {
+		result := callOpFunc(fnval, ctx, data, funcForm)
+		predicate := result.Bool()
+		if !predicate {
+			return nil
+		}
+		return data
+	}), nil
+}
+
+// MapFunc returns an unary function which applies the user-defined function which
+// maps, one-to-one, the incomfing value to a new value.  The user-defined function
+// must be of type:
+//   func(T) R - where T is the incoming item, R is the type of the returned mapped item
+func MapFunc(f interface{}) (operators.UnFunc, error) {
+	return ProcessFunc(f)
+}
+
+// FlatMapFunc returns an unary function which applies a user-defined function which
+// takes incoming comsite items and deconstruct them into individual items which can
+// then be re-streamed.  The type for the user-defined function is:
+//   func (T) R - where R is the original item, R is a slice of decostructed items
+// The slice returned should be restreamed by placing each item onto the stream for
+// downstream processing.
+func FlatMapFunc(f interface{}) (operators.UnFunc, error) {
+	fntype := reflect.TypeOf(f)
+
+	funcForm, err := isUnaryFuncForm(fntype)
+	if err != nil {
+		return nil, err
+	}
+	if funcForm == unaryFuncUnsupported {
+		return nil, fmt.Errorf("unsupported unary func type")
+	}
+
+	if fntype.Out(0).Kind() != reflect.Slice {
+		return nil, fmt.Errorf("unary FlatMap func must return slice")
+	}
+
+	fnval := reflect.ValueOf(f)
+	return operators.UnFunc(func(ctx context.Context, data interface{}) interface{} {
+		result := callOpFunc(fnval, ctx, data, funcForm)
+		return result.Interface()
+	}), nil
+}
+
+// isUnaryFuncForm ensures ftype is of supported function of
+// form func(in) out or func(context, in) out
+func isUnaryFuncForm(ftype reflect.Type) (unaryFuncForm, error) {
+	if ftype.NumOut() != 1 {
+		return unaryFuncUnsupported, fmt.Errorf("unary func must return one param")
+	}
+
+	switch ftype.Kind() {
+	case reflect.Func:
+		switch ftype.NumIn() {
+		case 1:
+			// f(in)out, ok
+			return unaryFuncForm1, nil
+		case 2:
+			// func(context,in)out
+			param0 := ftype.In(0)
+			if param0.Kind() != reflect.Interface {
+				return unaryFuncUnsupported, fmt.Errorf("unary must be type func(T)R or func(context.Context, T)R")
+			}
+			return unaryFuncForm2, nil
+		}
+	}
+	return unaryFuncUnsupported, fmt.Errorf("unary func must be of type func(T)R or func(context.Context,T)R")
+}
+
+func callOpFunc(fnval reflect.Value, ctx context.Context, data interface{}, funcForm unaryFuncForm) reflect.Value {
+	var result reflect.Value
+	switch funcForm {
+	case unaryFuncForm1:
+		arg0 := reflect.ValueOf(data)
+		result = fnval.Call([]reflect.Value{arg0})[0]
+	case unaryFuncForm2:
+		arg0 := reflect.ValueOf(ctx)
+		arg1 := reflect.ValueOf(data)
+		if !arg0.IsValid() {
+			arg0 = reflect.ValueOf(context.Background())
+		}
+		result = fnval.Call([]reflect.Value{arg0, arg1})[0]
+	}
+	return result
+}
+
+func isArgContext(val reflect.Value) bool {
+	_, ok := val.Interface().(context.Context)
+	return ok
+}

xstream/operators/operations.go

@@ -0,0 +1,191 @@
+package operators
+
+import (
+	"context"
+	"engine/common"
+	"fmt"
+	"sync"
+)
+
+var log = common.Log
+
+// UnOperation interface represents unary operations (i.e. Map, Filter, etc)
+type UnOperation interface {
+	Apply(ctx context.Context, data interface{}) interface{}
+}
+
+// UnFunc implements UnOperation as type func (context.Context, interface{})
+type UnFunc func(context.Context, interface{}) interface{}
+
+// Apply implements UnOperation.Apply method
+func (f UnFunc) Apply(ctx context.Context, data interface{}) interface{} {
+	return f(ctx, data)
+}
+
+
+
+type UnaryOperator struct {
+	op          UnOperation
+	concurrency int
+	input       chan interface{}
+	outputs     map[string]chan<- interface{}
+	mutex       sync.RWMutex
+	cancelled   bool
+	name 		string
+}
+
+// NewUnary creates *UnaryOperator value
+func New(name string) *UnaryOperator {
+	// extract logger
+	o := new(UnaryOperator)
+
+	o.concurrency = 1
+	o.input = make(chan interface{}, 1024)
+	o.outputs = make(map[string]chan<- interface{})
+	o.name = name
+	return o
+}
+
+func (o *UnaryOperator) GetName() string {
+	return o.name
+}
+
+// SetOperation sets the executor operation
+func (o *UnaryOperator) SetOperation(op UnOperation) {
+	o.op = op
+}
+
+// SetConcurrency sets the concurrency level for the operation
+func (o *UnaryOperator) SetConcurrency(concurr int) {
+	o.concurrency = concurr
+	if o.concurrency < 1 {
+		o.concurrency = 1
+	}
+}
+
+func (o *UnaryOperator) AddOutput(output chan<- interface{}, name string) {
+	if _, ok := o.outputs[name]; !ok{
+		o.outputs[name] = output
+	}else{
+		log.Error("fail to add output %s, operator %s already has an output of the same name", name, o.name)
+	}
+}
+
+func (o *UnaryOperator) GetInput() (chan<- interface{}, string) {
+	return o.input, o.name
+}
+
+// Exec is the entry point for the executor
+func (o *UnaryOperator) Exec(ctx context.Context) (err error) {
+
+	log.Printf("Unary operator %s is started.\n", o.name)
+
+	if len(o.outputs) <= 0 {
+		err = fmt.Errorf("No output channel found")
+		return
+	}
+
+	// validate p
+	if o.concurrency < 1 {
+		o.concurrency = 1
+	}
+
+	go func() {
+		var barrier sync.WaitGroup
+		wgDelta := o.concurrency
+		barrier.Add(wgDelta)
+
+		for i := 0; i < o.concurrency; i++ { // workers
+			go func(wg *sync.WaitGroup) {
+				defer wg.Done()
+				o.doOp(ctx)
+			}(&barrier)
+		}
+
+		wait := make(chan struct{})
+		go func() {
+			defer close(wait)
+			barrier.Wait()
+		}()
+
+		select {
+		case <-wait:
+			if o.cancelled {
+				log.Printf("Component cancelling...")
+				return
+			}
+		case <-ctx.Done():
+			log.Print("UnaryOp %s done.", o.name)
+			return
+		}
+	}()
+
+	return nil
+}
+
+func (o *UnaryOperator) doOp(ctx context.Context) {
+	if o.op == nil {
+		log.Println("Unary operator missing operation")
+		return
+	}
+	exeCtx, cancel := context.WithCancel(ctx)
+
+	defer func() {
+		log.Println("unary operator done, cancelling future items")
+		cancel()
+	}()
+
+	for {
+		select {
+		// process incoming item
+		case item, opened := <-o.input:
+			if !opened {
+				return
+			}
+
+			result := o.op.Apply(exeCtx, item)
+
+			switch val := result.(type) {
+			case nil:
+				continue
+			//case api.StreamError:
+			//	fmt.Println( val)
+			//	fmt.Println( val)
+			//	if item := val.Item(); item != nil {
+			//		select {
+			//		case o.output <- *item:
+			//		case <-exeCtx.Done():
+			//			return
+			//		}
+			//	}
+			//	continue
+			//case api.PanicStreamError:
+			//	util.Logfn(o.logf, val)
+			//	autoctx.Err(o.errf, api.StreamError(val))
+			//	panic(val)
+			//case api.CancelStreamError:
+			//	util.Logfn(o.logf, val)
+			//	autoctx.Err(o.errf, api.StreamError(val))
+			//	return
+			case error:
+				log.Println(val)
+				log.Println(val.Error())
+				continue
+
+			default:
+				for _, output := range o.outputs{
+					output <- val
+				}
+			}
+
+		// is cancelling
+		case <-exeCtx.Done():
+			log.Println("Cancelling....")
+			o.mutex.Lock()
+			cancel()
+			o.cancelled = true
+			o.mutex.Unlock()
+			return
+		}
+	}
+}

xstream/operators/window_op.go

@@ -0,0 +1,197 @@
+package operators
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"math"
+	"time"
+)
+
+type WindowType int
+const (
+	NO_WINDOW WindowType = iota
+	TUMBLING_WINDOW
+	HOPPING_WINDOW
+	SLIDING_WINDOW
+	SESSION_WINDOW
+)
+
+type WindowConfig struct {
+	Type WindowType
+	Length int64
+	Interval int64   //If interval is not set, it is equals to Length
+}
+
+type WindowOperator struct {
+	input       chan interface{}
+	outputs     map[string]chan<- interface{}
+	name 		string
+	ticker 		*time.Ticker
+	window      *WindowConfig
+	interval	int64
+	triggerTime int64
+}
+
+func NewWindowOp(name string, config *WindowConfig) *WindowOperator {
+	o := new(WindowOperator)
+
+	o.input = make(chan interface{}, 1024)
+	o.outputs = make(map[string]chan<- interface{})
+	o.name = name
+	o.window = config
+	switch config.Type{
+	case NO_WINDOW:
+	case TUMBLING_WINDOW:
+		o.ticker = time.NewTicker(time.Duration(config.Length) * time.Millisecond)
+		o.interval = config.Length
+	case HOPPING_WINDOW:
+		o.ticker = time.NewTicker(time.Duration(config.Interval) * time.Millisecond)
+		o.interval = config.Interval
+	case SLIDING_WINDOW:
+		o.interval = config.Length
+	case SESSION_WINDOW:
+		o.interval = config.Interval
+	default:
+		log.Errorf("Unsupported window type %d", config.Type)
+	}
+
+	return o
+}
+
+func (o *WindowOperator) GetName() string {
+	return o.name
+}
+
+func (o *WindowOperator) AddOutput(output chan<- interface{}, name string) {
+	if _, ok := o.outputs[name]; !ok{
+		o.outputs[name] = output
+	}else{
+		log.Error("fail to add output %s, operator %s already has an output of the same name", name, o.name)
+	}
+}
+
+func (o *WindowOperator) GetInput() (chan<- interface{}, string) {
+	return o.input, o.name
+}
+
+// Exec is the entry point for the executor
+func (o *WindowOperator) Exec(ctx context.Context) (err error) {
+
+	log.Printf("Window operator %s is started.\n", o.name)
+
+	if len(o.outputs) <= 0 {
+		err = fmt.Errorf("no output channel found")
+		return
+	}
+
+	go func() {
+		var (
+			inputs []*xsql.Tuple
+			c <-chan time.Time
+			timeoutTicker *time.Timer
+			timeout <-chan time.Time
+		)
+
+		if o.ticker != nil {
+			c = o.ticker.C
+		}
+
+		for {
+			select {
+			// process incoming item
+			case item, opened := <-o.input:
+				if !opened {
+					return
+				}
+				if d, ok := item.(*xsql.Tuple); !ok {
+					log.Errorf("Expect xsql.Tuple type.\n")
+					return
+				}else{
+					inputs = append(inputs, d)
+					switch o.window.Type{
+					case NO_WINDOW:
+						inputs = o.trigger(inputs, d.Timestamp)
+					case SLIDING_WINDOW:
+						inputs = o.trigger(inputs, d.Timestamp)
+					case SESSION_WINDOW:
+						if o.ticker == nil{ //Stopped by timeout or init
+							o.ticker = time.NewTicker(time.Duration(o.window.Length) * time.Millisecond)
+							c = o.ticker.C
+						}
+						if timeoutTicker != nil {
+							timeoutTicker.Stop()
+							timeoutTicker.Reset(time.Duration(o.window.Interval) * time.Millisecond)
+						} else {
+							timeoutTicker = time.NewTimer(time.Duration(o.window.Interval) * time.Millisecond)
+							timeout = timeoutTicker.C
+						}
+					}
+				}
+			case now := <-c:
+				if len(inputs) > 0 {
+					log.Infof("triggered by ticker")
+					inputs = o.trigger(inputs, common.TimeToUnixMilli(now))
+				}
+			case now := <-timeout:
+				if len(inputs) > 0 {
+					log.Infof("triggered by timeout")
+					inputs = o.trigger(inputs, common.TimeToUnixMilli(now))
+				}
+				o.ticker.Stop()
+				o.ticker = nil
+			// is cancelling
+			case <-ctx.Done():
+				log.Println("Cancelling....")
+				o.ticker.Stop()
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (o *WindowOperator) trigger(inputs []*xsql.Tuple, triggerTime int64) []*xsql.Tuple{
+	log.Printf("window %s triggered at %s", o.name, triggerTime)
+	var delta int64
+	if o.window.Type == HOPPING_WINDOW || o.window.Type == SLIDING_WINDOW {
+		lastTriggerTime := o.triggerTime
+		o.triggerTime = triggerTime
+		if lastTriggerTime <= 0 {
+			delta = math.MaxInt32  //max int, all events for the initial window
+		}else{
+			delta = o.triggerTime - lastTriggerTime - o.window.Interval
+			if delta > 100 && o.window.Interval > 0 {
+				log.Warnf("Possible long computation in window; Previous eviction time: %d, current eviction time: %d", lastTriggerTime, o.triggerTime)
+			}
+		}
+	}
+	var results xsql.MultiEmitterTuples = make([]xsql.EmitterTuples, 0)
+	i := 0
+	//Sync table
+	for _, tuple := range inputs{
+		if o.window.Type == HOPPING_WINDOW || o.window.Type == SLIDING_WINDOW {
+			diff := o.triggerTime - tuple.Timestamp
+			if diff >= o.window.Length + delta {
+				log.Infof("diff: %d, length: %d, delta: %d", diff, o.window.Length, delta)
+				log.Infof("tuple %s emitted at %d expired", tuple, tuple.Timestamp)
+				//Expired tuple, remove it by not adding back to inputs
+				continue
+			}
+			//All tuples in tumbling window are not added back
+			inputs[i] = tuple
+			i++
+		}
+		results.AddTuple(tuple)
+	}
+	if len(results) > 0{
+		log.Printf("window %s triggered for %d tuples", o.name, len(results))
+		for _, output := range o.outputs{
+			output <- results
+		}
+	}
+
+	return inputs[:i]
+}

xstream/streams.go

@@ -0,0 +1,129 @@
+package xstream
+
+import (
+	"context"
+	"engine/common"
+	"engine/xstream/operators"
+)
+
+var log = common.Log
+
+type TopologyNew struct {
+	sources []Source
+	sinks []Sink
+	ctx context.Context
+
+	drain chan error
+	ops []Operator
+}
+
+func New() (*TopologyNew) {
+	tp := &TopologyNew{}
+	return tp
+}
+
+func (tp *TopologyNew) AddSrc(src Source) (*TopologyNew) {
+	tp.sources = append(tp.sources, src)
+	return tp
+}
+
+func (tp *TopologyNew) AddSink(inputs []Emitter, snk Sink) (*TopologyNew) {
+	for _, input := range inputs{
+		input.AddOutput(snk.GetInput())
+	}
+	tp.sinks = append(tp.sinks, snk)
+	return tp
+}
+
+func (tp *TopologyNew) AddOperator(inputs []Emitter, operator Operator) (*TopologyNew) {
+	for _, input := range inputs{
+		input.AddOutput(operator.GetInput())
+	}
+	tp.ops = append(tp.ops, operator)
+	return tp
+}
+
+func Transform(op operators.UnOperation, name string) *operators.UnaryOperator {
+	operator := operators.New(name)
+	operator.SetOperation(op)
+	return operator
+}
+
+func (tp *TopologyNew) Map(f interface{}) (*TopologyNew){
+	op, err := MapFunc(f)
+	if err != nil {
+		log.Println(err)
+	}
+	return tp.Transform(op)
+}
+
+// Filter takes a predicate user-defined func that filters the stream.
+// The specified function must be of type:
+//   func (T) bool
+// If the func returns true, current item continues downstream.
+func (s *TopologyNew) Filter(f interface{}) *TopologyNew {
+	op, err := FilterFunc(f)
+	if err != nil {
+		s.drainErr(err)
+	}
+	return s.Transform(op)
+}
+
+// Transform is the base method used to apply transfomrmative
+// unary operations to streamed elements (i.e. filter, map, etc)
+// It is exposed here for completeness, use the other more specific methods.
+func (s *TopologyNew) Transform(op operators.UnOperation) *TopologyNew {
+	operator := operators.New("default")
+	operator.SetOperation(op)
+	s.ops = append(s.ops, operator)
+	return s
+}
+
+// prepareContext setups internal context before
+// stream starts execution.
+func (s *TopologyNew) prepareContext() {
+	if s.ctx == nil {
+		s.ctx = context.TODO()
+	}
+}
+
+func (s *TopologyNew) drainErr(err error) {
+	go func() { s.drain <- err }()
+}
+
+func (s *TopologyNew) Open() <-chan error {
+	s.prepareContext() // ensure context is set
+
+	log.Println("Opening stream")
+
+	// open stream
+	go func() {
+		// open source, if err bail
+		for _, src := range s.sources{
+			if err := src.Open(s.ctx); err != nil {
+				s.drainErr(err)
+				return
+			}
+		}
+
+		//apply operators, if err bail
+		for _, op := range s.ops {
+			if err := op.Exec(s.ctx); err != nil {
+				s.drainErr(err)
+				return
+			}
+		}
+
+		// open stream sink, after log sink is ready.
+		for _, snk := range s.sinks{
+			select {
+			case err := <-snk.Open(s.ctx):
+				log.Println("Closing stream")
+				s.drain <- err
+			}
+		}
+
+	}()
+
+	return s.drain
+}

xstream/test/testconf.json

@@ -0,0 +1,21 @@
+{
+  "servers": {
+    "srv1": {
+      "addr": "127.0.0.1",
+      "port": 1883,
+      "tls": false,
+      "clientId": ""
+    },
+    "srv2": {
+      "addr": "10.211.55.6",
+      "port": 1883,
+      "tls": false,
+      "clientId": "client1"
+    }
+  },
+
+  "conf_string":"test",
+  "conf_int": 10,
+  "conf_float": 32.3,
+  "conf_bool": true
+}

xstream/types.go

@@ -0,0 +1,33 @@
+package xstream
+
+import (
+	"context"
+)
+
+type Emitter interface {
+	AddOutput(chan<- interface{}, string)
+}
+
+type Source interface {
+	Emitter
+	Open(context context.Context) error
+}
+
+type Collector interface {
+	GetInput() (chan<- interface{}, string)
+}
+
+type Sink interface {
+	Collector
+	Open(context context.Context) <-chan error
+}
+
+type Operator interface{
+	Emitter
+	Collector
+	Exec(context context.Context) error
+}
+
+type TopNode interface{
+	GetName() string
+}

xstream/util.go

@@ -0,0 +1,131 @@
+package xstream
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type Conf map[string]interface{}
+
+var confs = make(map[string] Conf)
+
+func GetConfAsString(file, key string) (string, error) {
+	val, err := getConfValue(file, key)
+
+	if err != nil {
+		return "", err
+	}
+
+	if v, ok := val.(string); ok {
+		return v, nil
+	} else if val == nil {
+		return "", nil
+	}else {
+		return "", fmt.Errorf("The value %s is not type of string for key %s.\n", val, key )
+	}
+}
+
+func GetConfAsInt(file, key string) (int, error) {
+	val, err := getConfValue(file, key)
+
+	if err != nil {
+		return 0, err
+	}
+
+	if v, ok := val.(float64); ok {
+		return int(v), nil
+	} else {
+		return 0, fmt.Errorf("The value {0} is not type of int for key {1}.\n", )
+	}
+}
+
+func GetConfAsFloat(file, key string) (float64, error) {
+	val, err := getConfValue(file, key)
+
+	if err != nil {
+		return 0, err
+	}
+
+	if v, ok := val.(float64); ok {
+		return v, nil
+	} else {
+		return 0, fmt.Errorf("The value {0} is not type of float for key {1}.\n", )
+	}
+}
+
+func GetConfAsBool(file, key string) (bool, error) {
+	val, err := getConfValue(file, key)
+
+	if err != nil {
+		return false, err
+	}
+
+	if v, ok := val.(bool); ok {
+		return v, nil
+	} else {
+		return false, fmt.Errorf("The value {0} is not type of bool for key {1}.\n", )
+	}
+}
+
+
+func getConfValue(file, key string) (interface{}, error) {
+	if conf, ok := confs[file]; !ok {
+		if c, e := initConf(file); e != nil {
+			return nil, e
+		} else {
+			confs[file] = c
+			return getValue(c, key)
+		}
+	} else {
+		return getValue(conf, key)
+	}
+}
+
+func initConf(file string) (Conf, error) {
+	conf := make(Conf)
+	fp, _ := filepath.Abs(file)
+	if f, err1 := os.Open(fp); err1 == nil {
+		defer f.Close()
+
+		byteValue, _ := ioutil.ReadAll(f)
+		if err2 := json.Unmarshal([]byte(byteValue), &conf); err2 != nil {
+			return nil, err2
+		}
+		log.Printf("Successfully to load the configuration file %s.\n", fp)
+	} else {
+		//Try as absolute path
+		if f, err1 := os.Open(file); err1 == nil {
+			byteValue, _ := ioutil.ReadAll(f)
+			if err2 := json.Unmarshal([]byte(byteValue), &conf); err2 != nil {
+				return nil, err2
+			}
+			log.Printf("Successfully to load the configuration file %s.\n", file)
+		} else {
+			return nil, fmt.Errorf("Cannot load configuration file %s.\n", file)
+		}
+	}
+	return conf, nil
+}
+
+func getValue(conf Conf, key string) (interface{}, error)  {
+	keys := strings.Split(key, ".")
+
+	if len(keys) == 1 {
+		return conf[key], nil
+	}
+
+	nkey := strings.Join(keys[1:], ".")
+	ckey := strings.Join(keys[0:1], "")
+
+	if c, ok := conf[ckey].(map[string]interface {}); ok {
+		return getValue(c, nkey)
+	} else {
+
+		return nil, fmt.Errorf("%s does not exsit for key %s.", conf, key)
+	}
+}
+

xstream/util_test.go

@@ -0,0 +1,38 @@
+package xstream
+
+import (
+	"testing"
+)
+
+func TestConf(t *testing.T) {
+	var file = "test/testconf.json"
+
+	if v, e := GetConfAsString(file, "conf_string"); (e != nil || (v != "test")) {
+		t.Errorf("Expect %s, actual %s; error is %s. \n", "test", v, e)
+	}
+
+	if v, e := GetConfAsInt(file, "conf_int"); (e != nil || (v != 10)) {
+		t.Errorf("Expect %s, actual %d. error is %s. \n ", "10", v, e)
+	}
+
+	if v, e := GetConfAsFloat(file, "conf_float"); (e != nil || (v != 32.3)) {
+		t.Errorf("Expect %s, actual %f. error is %s. \n ", "32.3", v, e)
+	}
+
+	if v, e := GetConfAsBool(file, "conf_bool"); (e != nil || (v != true)) {
+		t.Errorf("Expect %s, actual %v. error is %s. \n", "true", v, e)
+	}
+
+	if v, e := GetConfAsString(file, "servers.srv1.addr"); (e != nil || (v != "127.0.0.1")) {
+		t.Errorf("Expect %s, actual %s. error is %s. \n", "127.0.0.1", v, e)
+	}
+
+	if v, e := GetConfAsString(file, "servers.srv1.clientid"); (e != nil || (v != "")) {
+		t.Errorf("Expect %s, actual %s. error is %s. \n", "", v, e)
+	}
+
+	if v, e := GetConfAsInt(file, "servers.srv2.port"); (e != nil || (v != 1883)) {
+		t.Errorf("Expect %s, actual %d. error is %s. \n", "1883", v, e)
+	}
+
+}