0.0.2 version

.gitignore

@@ -13,5 +13,12 @@
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
 
+data/*
+log/*
+
 .git/
 .idea/
+
+go.sum
+_build
+_packages

Makefile

@@ -0,0 +1,43 @@
+BUILD_PATH ?= _build
+PACKAGES_PATH ?= _packages
+
+GO111MODULE ?= 
+GOPROXY ?= https://goproxy.io
+
+GOOS ?= ""
+GOARCH ?= ""
+
+.PHONY: build
+build:
+	@mkdir -p $(BUILD_PATH)/engine/bin
+	@mkdir -p $(BUILD_PATH)/engine/etc
+	@mkdir -p $(BUILD_PATH)/engine/data
+	@mkdir -p $(BUILD_PATH)/engine/plugins
+	@mkdir -p $(BUILD_PATH)/engine/log
+
+	@cp -r etc/* $(BUILD_PATH)/engine/etc
+
+	@if [ ! -z $(GOOS) ] && [ ! -z $(GOARCH) ];then \
+		GO111MODULE=on GOPROXY=https://goproxy.io GOOS=$(GOOS) $(GOARCH)=$(GOARCH) CGO_ENABLED=0 go build -ldflags="-s -w" -o cli xstream/cli/main.go; \
+		GO111MODULE=on GOPROXY=https://goproxy.io GOOS=$(GOOS) $(GOARCH)=$(GOARCH) CGO_ENABLED=0 go build -ldflags="-s -w" -o server xstream/server/main.go; \
+	else \
+		GO111MODULE=on GOPROXY=https://goproxy.io CGO_ENABLED=0 go build -ldflags="-s -w" -o cli xstream/cli/main.go; \
+		GO111MODULE=on GOPROXY=https://goproxy.io CGO_ENABLED=0 go build -ldflags="-s -w" -o server xstream/server/main.go; \
+	fi
+	@if [ ! -z $$(which upx) ]; then upx ./cli; upx ./server; fi
+	@mv ./cli ./server $(BUILD_PATH)/engine/bin
+	@echo "Build successfully"
+
+.PHONY: pkg
+pkg: build
+	@mkdir -p $(PACKAGES_PATH)
+	@if [ ! -z $(GOOS) ] && [ ! -z $(GOARCH) ];then \
+		package_name=engine_$(GOARCH); \
+	else \
+		package_name=engine; \
+	fi; \
+	cd $(BUILD_PATH); \
+	zip -rq $${package_name}.zip engine; \
+	tar -czf $${package_name}.tar.gz engine; \
+	mv engine.zip engine.tar.gz ../$(PACKAGES_PATH)
+	@echo "Package build success"

README.md

@@ -1,8 +1,50 @@
-# edge_rule_engine
+# Rule Engine for Edge
+
+## Highlight
 
-#### Introduction
 A SQL based lightweight IoT streaming rule engine running at resource constrained edge devices.
+- Native run with small overhead ( ~13MB package), support Linux/Windows/Mac OS
 - SQL based, easy to use
-- Native run with small overhead 
-- Capability of consuming different source (preferred is MQTT)
+- Built-in support for MQTT source
+- Extension - user can customize the rule engine
+- RESTful APIs for rules management
+
+## Document
+
+- [Getting started](docs/getting_started.md)
+- [Reference guide](docs/index.md)
+
+## Build from source code
+
+#### Prepare
+
++ Go version >= 1.11
+
+#### Build binary file
+
++ Build binary file
+
+  ```shell
+  $ make
+  ```
+
++ Cross build binary file
+
+  ```shell
+  $ GOOS=linux GOARCH=arm make 
+  ```
+
+#### Get the compressed file
+
++ Get the compressed files
+ 
+  ```
+  $ make pkg
+  ```
+
++ Get the cross-build compressed file
 
+  ```
+  $ GOOS=linux GOARCH=arm make pkg
+  ```
+  

common/data.go

@@ -0,0 +1,160 @@
+package common
+
+import (
+	"errors"
+	"time"
+)
+
+type Rule struct {
+	Name, Json string
+}
+
+/**** Timer Mock *******/
+
+/** Ticker **/
+type Ticker interface {
+	GetC() <-chan time.Time
+	Stop()
+	Trigger(ti int64)
+}
+
+type DefaultTicker struct{
+	time.Ticker
+}
+
+func NewDefaultTicker(d int) *DefaultTicker{
+	return &DefaultTicker{*(time.NewTicker(time.Duration(d) * time.Millisecond))}
+}
+
+func (t *DefaultTicker) GetC() <-chan time.Time{
+	return t.C
+}
+
+func (t *DefaultTicker) Trigger(ti int64) {
+	Log.Fatal("ticker trigger unsupported")
+}
+
+type MockTicker struct {
+	c chan time.Time
+	duration int64
+	lastTick int64
+}
+
+func NewMockTicker(d int) *MockTicker{
+	if d <= 0 {
+		panic(errors.New("non-positive interval for MockTicker"))
+	}
+	c := make(chan time.Time, 1)
+	t := &MockTicker{
+		c: c,
+		duration: int64(d),
+		lastTick: GetMockNow(),
+	}
+	return t
+}
+
+func (t *MockTicker) SetDuration(d int){
+	t.duration = int64(d)
+	t.lastTick = GetMockNow()
+}
+
+func (t *MockTicker) GetC() <-chan time.Time{
+	return t.c
+}
+
+func (t *MockTicker) Stop() {
+	//do nothing
+}
+
+func (t *MockTicker) Trigger(ti int64) {
+	t.lastTick = ti
+	t.c <- time.Unix(ti/1000, ti%1000*1e6)
+}
+
+func (t *MockTicker) DoTick(c int64) {
+	Log.Infof("do tick at %d, last tick %d", c, t.lastTick)
+	if t.lastTick == 0 {
+		t.lastTick = c
+	}
+	if c >= (t.lastTick + t.duration){
+		Log.Info("trigger tick")
+		t.Trigger(t.lastTick + t.duration)
+	}
+}
+
+/** Timer **/
+type Timer interface {
+	GetC() <-chan time.Time
+	Stop() bool
+	Reset(d time.Duration) bool
+	Trigger(ti int64)
+}
+
+type DefaultTimer struct{
+	time.Timer
+}
+
+func NewDefaultTimer(d int) *DefaultTimer{
+	return &DefaultTimer{*(time.NewTimer(time.Duration(d) * time.Millisecond))}
+}
+
+func (t *DefaultTimer) GetC() <-chan time.Time{
+	return t.C
+}
+
+func (t *DefaultTimer) Trigger(ti int64) {
+	Log.Fatal("timer trigger unsupported")
+}
+
+type MockTimer struct {
+	c chan time.Time
+	duration int64
+	createdAt int64
+}
+
+func NewMockTimer(d int) *MockTimer{
+	if d <= 0 {
+		panic(errors.New("non-positive interval for MockTimer"))
+	}
+	c := make(chan time.Time, 1)
+	t := &MockTimer{
+		c: c,
+		duration: int64(d),
+		createdAt: GetMockNow(),
+	}
+	return t
+}
+
+func (t *MockTimer) GetC() <-chan time.Time{
+	return t.c
+}
+
+func (t *MockTimer) Stop() bool{
+	t.createdAt = 0
+	return true
+}
+
+func (t *MockTimer) SetDuration(d int){
+	t.duration = int64(d)
+	t.createdAt = GetMockNow()
+	Log.Infoln("reset timer created at %v", t.createdAt)
+}
+
+func (t *MockTimer) Reset(d time.Duration) bool{
+	Log.Infoln("reset timer")
+	t.SetDuration(int(d.Nanoseconds()/1e6))
+	return true
+}
+
+func (t *MockTimer) Trigger(ti int64) {
+	t.c <- time.Unix(ti/1000, ti%1000*1e6)
+	t.createdAt = 0
+}
+
+func (t *MockTimer) DoTick(c int64) {
+	Log.Infof("do tick at %d, created at", c, t.createdAt)
+	if t.createdAt > 0 && c >= (t.createdAt + t.duration){
+		Log.Info("trigger timer")
+		t.Trigger(t.createdAt + t.duration)
+	}
+}

common/time_util.go

@@ -0,0 +1,343 @@
+package common
+
+import (
+	"fmt"
+	"time"
+)
+
+//var (
+//	formats = map[string]string{
+//		"MMMM": "January",
+//		"MMM":  "Jan",
+//		"MM":   "01",
+//		"M":    "1",
+//		"YYYY": "2006",
+//		"yyyy": "2006",
+//		"YY":   "06",
+//		"yy":   "06",
+//		"G":    "AD",
+//		"EEEE": "Monday",
+//		"EEE":  "Mon",
+//		"dd":   "02",
+//		"d":    "2",
+//		"HH":   "15",
+//		"hh":   "03",
+//		"h":    "3",
+//		"mm":   "04",
+//		"m":    "4",
+//		"ss":   "05",
+//		"s":    "5",
+//		"a":    "PM",
+//		"S":    ".0",
+//		"SS":    ".00",
+//		"SSS":   ".000",
+//		"SSSN":  ".0000",
+//		"SSSNN": ".00000",
+//		"SSSNNN": ".000000",
+//		"SSSNNNN": ".0000000",
+//		"SSSNNNNN":".00000000",
+//		"SSSNNNNNN":".000000000",
+//		"z":    "MST",
+//		"Z":    "-0700",
+//		"X":    "-07",
+//		"XX":    "-0700",
+//		"XXX":  "-07:00",
+//	}
+//)
+
+const JSISO = "2006-01-02T15:04:05.000Z07:00"
+const ISO8601 = "2006-01-02T15:04:05"
+
+func TimeToUnixMilli(time time.Time) int64 {
+	return time.UnixNano() / 1e6
+}
+
+func InterfaceToUnixMilli(i interface{}, format string) (int64, error) {
+	switch t := i.(type) {
+	case int64:
+		return t, nil
+	case int:
+		return int64(t), nil
+	case float64:
+		return int64(t), nil
+	case time.Time:
+		return TimeToUnixMilli(t), nil
+	case string:
+		var ti time.Time
+		var err error
+		var f = JSISO
+		if format != ""{
+			f, err = convertFormat(format)
+			if err != nil{
+				return 0, err
+			}
+		}
+		ti, err = time.Parse(f, t)
+		if err != nil{
+			return 0, err
+		}
+		return TimeToUnixMilli(ti), nil
+	default:
+		return 0, fmt.Errorf("unsupported type to convert to timestamp %v", t)
+	}
+}
+
+func InterfaceToTime(i interface{}, format string) (time.Time, error) {
+	switch t := i.(type) {
+	case int64:
+		return TimeFromUnixMilli(t), nil
+	case int:
+		return TimeFromUnixMilli(int64(t)), nil
+	case float64:
+		return TimeFromUnixMilli(int64(t)), nil
+	case time.Time:
+		return t, nil
+	case string:
+		var ti time.Time
+		var err error
+		var f = JSISO
+		if format != ""{
+			f, err = convertFormat(format)
+			if err != nil{
+				return ti, err
+			}
+		}
+		ti, err = time.Parse(f, t)
+		if err != nil{
+			return ti, err
+		}
+		return ti, nil
+	default:
+		return time.Now(), fmt.Errorf("unsupported type to convert to timestamp %v", t)
+	}
+}
+
+func TimeFromUnixMilli(t int64) time.Time {
+	return time.Unix(t/1000, t%1000).UTC()
+}
+
+func ParseTime(t string, f string) (time.Time, error){
+	if f, err := convertFormat(f); err != nil{
+		return time.Now(), err
+	}else{
+		return time.Parse(f, t)
+	}
+}
+
+func FormatTime(time time.Time, f string) (string, error){
+	if f, err := convertFormat(f); err != nil{
+		return "", err
+	}else{
+		return time.Format(f), nil
+	}
+}
+
+//func convertFormat(f string) string {
+//	re := regexp.MustCompile(`(?m)(M{4})|(M{3})|(M{2})|(M{1})|(Y{4})|(Y{2})|(y{4})|(y{2})|(G{1})|(E{4})|(E{3})|(d{2})|(d{1})|(H{2})|(h{2})|(h{1})|(m{2})|(m{1})|(s{2})|(s{1})|(a{1})|(S{3}N{6})|(S{3}N{5})|(S{3}N{4})|(S{3}N{3})|(S{3}N{2})|(S{3}N{1})|(S{3})|(S{2})|(S{1})|(z{1})|(Z{1})|(X{3})|(X{2})|(X{1})`)
+//	for _, match := range re.FindAllString(f, -1) {
+//		for key, val := range formats {
+//			if match == key {
+//				f = strings.Replace(f, match, val, -1)
+//			}
+//		}
+//	}
+//	return f
+//}
+
+func convertFormat(f string) (string, error){
+	formatRune := []rune(f)
+	lenFormat := len(formatRune)
+	out := ""
+	for i := 0; i < len(formatRune); i++ {
+		switch r := formatRune[i]; r {
+		case 'Y', 'y':
+			j := 1
+			for ; i+j < lenFormat && j<=4; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 4: // YYYY
+				out += "2006"
+			case 2: // YY
+				out += "06"
+			default:
+				return "", fmt.Errorf("invalid time format %s for Y/y", f)
+			}
+		case 'G': //era
+			out += "AD"
+		case 'M': // M MM MMM MMMM month of year
+			j := 1
+			for ; i+j < lenFormat && j<=4; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // M
+				out += "1"
+			case 2: // MM
+				out += "01"
+			case 3: // MMM
+				out += "Jan"
+			case 4: // MMMM
+				out += "January"
+			}
+		case 'd': // d dd day of month
+			j := 1
+			for ; i+j < lenFormat && j<=2; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // d
+				out += "2"
+			case 2: // dd
+				out += "02"
+			}
+		case 'E': // M MM MMM MMMM month of year
+			j := 1
+			for ; i+j < lenFormat && j<=4; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 3: // EEE
+				out += "Mon"
+			case 4: // EEEE
+				out += "Monday"
+			default:
+				return "", fmt.Errorf("invalid time format %s for E", f)
+			}
+		case 'H': // HH
+			j := 1
+			for ; i+j < lenFormat && j<=2; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+
+			}
+			i = i + j - 1
+			switch j {
+			case 2: // HH
+				out += "15"
+			default:
+				return "", fmt.Errorf("invalid time format %s of H, only HH is supported", f)
+			}
+		case 'h': // h hh
+			j := 1
+			for ; i+j < lenFormat && j<=2; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 1:  // h
+				out += "3"
+			case 2: // hh
+				out += "03"
+			}
+		case 'a': // a
+			out += "PM"
+		case 'm': // m mm minute of hour
+			j := 1
+			for ; i+j < lenFormat && j <= 2; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // m
+				out += "4"
+			case 2: // mm
+				out += "04"
+			}
+		case 's': // s ss
+			j := 1
+			for ; i+j < lenFormat && j<=2 ; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // s
+				out += "5"
+			case 2: // ss
+				out += "05"
+			}
+
+		case 'S': // S SS SSS
+			j := 1
+			for ; i+j < lenFormat && j<=3; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // S
+				out += ".0"
+			case 2: // SS
+				out += ".00"
+			case 3: // SSS
+				out += ".000"
+			}
+		case 'z': // z
+			out += "MST"
+		case 'Z': // Z
+			out += "-0700"
+		case 'X': // X XX XXX
+			j := 1
+			for ; i+j < lenFormat && j<=3; j++ {
+				if formatRune[i+j] != r {
+					break
+				}
+			}
+			i = i + j - 1
+			switch j {
+			case 1: // X
+				out += "-07"
+			case 2: // XX
+				out += "-0700"
+			case 3: // XXX
+				out += "-07:00"
+			}
+		case '\'': // ' (text delimiter)  or '' (real quote)
+
+			// real quote
+			if formatRune[i+1] == r {
+				out += "'"
+				i = i + 1
+				continue
+			}
+
+			tmp := []rune{}
+			j := 1
+			for ; i+j < lenFormat; j++ {
+				if formatRune[i+j] != r {
+					tmp = append(tmp, formatRune[i+j])
+					continue
+				}
+				break
+			}
+			i = i + j
+			out += string(tmp)
+		default:
+			out += string(r)
+		}
+	}
+	return out, nil
+}

common/util.go

@@ -2,22 +2,32 @@ package common
 
 import (
 	"bytes"
-	"errors"
-	"flag"
+	"context"
 	"fmt"
 	"github.com/dgraph-io/badger"
+	"github.com/go-yaml/yaml"
 	"github.com/sirupsen/logrus"
+	"io/ioutil"
 	"os"
-	"time"
+	"path/filepath"
 )
 
-const LogLocation = "stream.log"
+const (
+	logFileName = "stream.log"
+	LoggerKey = "logger"
+	etc_dir = "/etc/"
+	data_dir = "/data/"
+	log_dir = "/log/"
+)
 
 var (
 	Log *logrus.Logger
-	Env string
-
+	Config *XStreamConf
+	IsTesting bool
 	logFile *os.File
+	mockTicker *MockTicker
+	mockTimer *MockTimer
+	mockNow int64
 )
 
 type logRedirect struct {
@@ -40,24 +50,74 @@ func (l *logRedirect) Debugf(f string, v ...interface{}) {
 	Log.Debug(fmt.Sprintf(f, v...))
 }
 
+func GetLogger(ctx context.Context) *logrus.Entry {
+	if ctx != nil{
+		l, ok := ctx.Value(LoggerKey).(*logrus.Entry)
+		if l != nil && ok {
+			return l
+		}
+	}
+	return Log.WithField("caller", "default")
+}
+
+func LoadConf(confName string) []byte {
+	confDir, err := GetConfLoc()
+	if err != nil {
+		Log.Fatal(err)
+	}
+
+	file := confDir + confName
+	b, err := ioutil.ReadFile(file)
+	if err != nil {
+		Log.Fatal(err)
+	}
+	return b
+}
+
+type XStreamConf struct {
+	Debug bool `yaml:"debug"`
+	Port int `yaml:"port"`
+}
+
+var StreamConf = "xstream.yaml"
+
 func init(){
-	flag.StringVar(&Env, "env", "dev", "set environment to prod or test")
-	flag.Parse()
 	Log = logrus.New()
-	if Env == "prod"{
-		logFile, err := os.OpenFile(LogLocation, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+	Log.SetFormatter(&logrus.TextFormatter{
+		DisableColors: true,
+		FullTimestamp: true,
+	})
+	b := LoadConf(StreamConf)
+	var cfg map[string]XStreamConf
+	if err := yaml.Unmarshal(b, &cfg); err != nil {
+		Log.Fatal(err)
+	}
+
+	if c, ok := cfg["basic"]; !ok{
+		Log.Fatal("no basic config in xstream.yaml")
+	}else{
+		Config = &c
+	}
+
+	if !Config.Debug {
+		logDir, err := GetLoc(log_dir)
+		if err != nil {
+			Log.Fatal(err)
+		}
+		file := logDir + logFileName
+		logFile, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
 		if err == nil {
 			Log.Out = logFile
 		} else {
 			Log.Infof("Failed to log to file, using default stderr")
 		}
+	}else{
+		Log.SetLevel(logrus.DebugLevel)
 	}
 }
 
 func DbOpen(dir string) (*badger.DB, error) {
-	opts := badger.DefaultOptions
-	opts.Dir = dir
-	opts.ValueDir = dir
+	opts := badger.DefaultOptions(dir)
 	opts.Logger = &logRedirect{}
 	db, err := badger.Open(opts)
 	return db, err
@@ -75,7 +135,7 @@ func DbSet(db *badger.DB, key string, value string) error {
 		if err != nil {
 			err = txn.Set([]byte(key), []byte(value))
 		}else{
-			err = errors.New(fmt.Sprintf("key %s already exist, delete it before creating a new one", key))
+			err = fmt.Errorf("key %s already exist, delete it before creating a new one", key)
 		}
 
 		return err
@@ -143,36 +203,109 @@ func CloseLogger(){
 	}
 }
 
-func GetConfLoc()(string, error) {
+func GetConfLoc()(string, error){
+	return GetLoc(etc_dir)
+}
+
+func GetDataLoc() (string, error) {
+	return GetLoc(data_dir)
+}
+
+func GetLoc(subdir string)(string, error) {
 	dir, err := os.Getwd()
 	if err != nil {
 		return "", err
 	}
-	confDir := dir + "/conf/"
+	confDir := dir + subdir
 	if _, err := os.Stat(confDir); os.IsNotExist(err) {
-		return "", err
+		lastdir := dir
+		for len(dir) > 0 {
+			dir = filepath.Dir(dir)
+			if lastdir == dir {
+				break
+			}
+			confDir = dir + subdir
+			if _, err := os.Stat(confDir); os.IsNotExist(err) {
+				lastdir = dir
+				continue
+			} else {
+				//Log.Printf("Trying to load file from %s", confDir)
+				return confDir, nil
+			}
+		}
+	} else {
+		//Log.Printf("Trying to load file from %s", confDir)
+		return confDir, nil
 	}
 
-	return confDir, nil
+	return "", fmt.Errorf("conf dir not found")
 }
 
-
-func GetDataLoc() (string, error) {
-	dir, err := os.Getwd()
-	if err != nil {
-		return "", err
+//Time related. For Mock
+func GetTicker(duration int) Ticker {
+	if IsTesting{
+		if mockTicker == nil{
+			mockTicker = NewMockTicker(duration)
+		}else{
+			mockTicker.SetDuration(duration)
+		}
+		return mockTicker
+	}else{
+		return NewDefaultTicker(duration)
 	}
-	dataDir := dir + "/data/"
+}
 
-	if _, err := os.Stat(dataDir); os.IsNotExist(err) {
-		if err := os.Mkdir(dataDir, os.ModePerm); err != nil {
-			return "", fmt.Errorf("Find error %s when trying to locate xstream data folder.\n", err)
+func GetTimer(duration int) Timer {
+	if IsTesting{
+		if mockTimer == nil{
+			mockTimer = NewMockTimer(duration)
+		}else{
+			mockTimer.SetDuration(duration)
 		}
+		return mockTimer
+	}else{
+		return NewDefaultTimer(duration)
+	}
+}
+
+
+/****** For Test Only ********/
+func GetMockTicker() *MockTicker{
+	return mockTicker
+}
+
+func ResetMockTicker(){
+	if mockTicker != nil{
+		mockTicker.lastTick = 0
 	}
+}
 
-	return dataDir, nil
+func GetMockTimer() *MockTimer{
+	return mockTimer
 }
 
-func TimeToUnixMilli(time time.Time) int64 {
-	return time.UnixNano() / 1e6;
-}
+func SetMockNow(now int64){
+	mockNow = now
+}
+
+func GetMockNow() int64{
+	return mockNow
+}
+
+/*********** Type Cast Utilities *****/
+//TODO datetime type
+func ToString(input interface{}) string{
+	return fmt.Sprintf("%v", input)
+}
+func ToInt(input interface{}) (int, error){
+	switch t := input.(type) {
+	case float64:
+		return int(t), nil
+	case int64:
+		return int(t), nil
+	case int:
+		return t, nil
+	default:
+		return 0, fmt.Errorf("unsupported type %T of %[1]v", input)
+	}
+}

docs/cli/overview.md

@@ -0,0 +1,10 @@
+The XStream CLI (command line interface) tools provides streams and rules management. 
+
+The XStream CLI acts as a client to the XStream server. The XStream server runs the engine that executes the stream or rule queries. This includes processing stream or rule definitions, manage rule status and io.
+
+*XStream CLI Architecture*
+![CLI Arch](resources/arch.png)
+
+- [Streams](streams.md)
+- [Rules](rules.md)
+

docs/cli/rules.md

@@ -0,0 +1,171 @@
+# Rules management
+
+The XStream rule command line tools allows you to manage rules, such as create, show, drop, describe, start, stop and restart rules. 
+
+## create a rule
+
+The command is used for creating a rule.  The rule's definition is specified with JSON format, read [rule](../rules/overview.md) for more detailed information.
+
+```shell
+create rule $rule_name $rule_json | create rule $rule_name -f $rule_def_file
+```
+
+The rule can be created with two ways. 
+
+- Specify the rule definition in command line.
+
+Sample:
+
+```shell
+# bin/cli create rule rule1 {"sql": "SELECT * from demo","actions": [{"log":  {}},{"mqtt":  {"server":"tcp://127.0.0.1:1883", "topic":"demoSink"}}]}
+```
+
+The command create a rule named ``rule1``. 
+
+- Specify the rule definition in file. If the rule is complex, or the rule is already wrote in text files with well organized formats, you can just specify the rule definition through ``-f`` option.
+
+Sample:
+
+```shell
+# bin/cli create rule rule1 -f /tmp/rule.txt
+```
+
+Below is the contents of ``rule.txt``.
+
+```json
+{
+  "sql": "SELECT * from demo",
+  "actions": [
+    {
+      "log": {}
+    },
+    {
+      "mqtt": {
+        "server": "tcp://127.0.0.1:1883",
+        "topic": "demoSink"
+      }
+    }
+  ]
+}
+```
+
+## show rules
+
+The command is used for displaying all of rules defined in the server.
+
+```shell
+show rules
+```
+
+Sample:
+
+```shell
+# bin/cli show rules
+rule1
+rule2
+```
+
+## describe a rule
+
+The command is used for print the detailed definition of rule.
+
+```shell
+describe rule $rule_name
+```
+
+Sample: 
+
+```shell
+# bin/cli describe rule rule1
+{
+  "sql": "SELECT * from demo",
+  "actions": [
+    {
+      "log": {}
+    },
+    {
+      "mqtt": {
+        "server": "tcp://127.0.0.1:1883",
+        "topic": "demoSink"
+      }
+    }
+  ]
+}
+```
+
+## drop a rule
+
+The command is used for drop the rule.
+
+```shell
+drop rule $rule_name
+```
+
+Sample:
+
+```shell
+# bin/cli drop rule rule1
+rule rule1 dropped
+```
+
+## start a rule
+
+The command is used to start running the rule.
+
+```shell
+start rule $rule_name
+```
+
+Sample:
+
+```shell
+# bin/cli start rule rule1
+rule rule1 started
+```
+
+## stop a rule
+
+The command is used to stop running the rule.
+
+```shell
+stop rule $rule_name
+```
+
+Sample:
+
+```shell
+# bin/cli stop rule rule1
+rule rule1 stopped
+```
+
+## restart a rule
+
+The command is used to restart the rule.
+
+```shell
+restart rule $rule_name
+```
+
+Sample:
+
+```shell
+# bin/cli restart rule rule1
+rule rule1 restarted
+```
+
+## get the status of a rule
+
+The command is used to get the status of the rule. The status can be
+- running
+- stopped: $reason
+
+```shell
+getstatus rule $rule_name
+```
+
+Sample:
+
+```shell
+# bin/cli getstatus rule rule1
+running
+```

docs/cli/streams.md

@@ -0,0 +1,118 @@
+# Streams management
+
+The XStream stream command line tools allows you to manage the streams, such as create, describe, show and drop stream definitions.
+
+## create a stream
+
+The command is used for creating a stream. For more detailed information of stream definition, please refer to [streams](../sqls/streams.md).
+
+```shell
+create stream $stream_name $stream_def | create stream -f $stream_def_file
+```
+
+- Specify the stream definition in command line.
+
+Sample:
+
+```shell
+# bin/cli create stream my_stream '(id bigint, name string, score float) WITH ( datasource = "topic/temperature", FORMAT = "json", KEY = "id");'
+stream my_stream created
+```
+
+The command create a rule named ``my_stream``. 
+
+- Specify the stream definition in file. If the stream is complex, or the stream is already wrote in text files with well organized formats, you can just specify the stream definition through ``-f`` option.
+
+Sample:
+
+```shell
+# bin/cli create stream -f /tmp/my_stream.txt
+stream my_stream created
+```
+
+Below is the contents of ``my_stream.txt``.
+
+```json
+my_stream(id bigint, name string, score float)
+    WITH ( datasource = "topic/temperature", FORMAT = "json", KEY = "id");
+```
+
+## show streams
+
+The command is used for displaying all of streams defined in the server.
+
+```shell
+show streams
+```
+
+Sample:
+
+```shell
+# bin/cli show streams
+my_stream
+```
+
+## describe a stream
+
+The command is used for print the detailed definition of stream.
+
+```shell
+describe stream $stream_name
+```
+
+Sample:
+
+```shell
+# bin/cli describe stream my_stream
+Fields
+--------------------------------------------------------------------------------
+id	bigint
+name	string
+score	float
+
+FORMAT: json
+KEY: id
+DATASOURCE: topic/temperature
+```
+
+## drop a stream
+
+The command is used for drop the stream definition.
+
+```shell
+drop stream $stream_name
+```
+
+Sample:
+
+```shell
+# bin/cli drop stream my_stream
+stream my_stream dropped
+```
+
+## query against streams
+The command is used for querying data from stream.  
+```
+query
+```
+
+Sample:
+
+```shell
+# bin/cli query
+xstream > 
+```
+
+After typing ``query`` sub-command, it prompts ``xstream > ``, then type SQLs (see [XStream SQL reference](../sqls/overview.md) for how to use XStream SQL) in the command prompt and press enter. 
+
+The results will be print in the console.
+
+```shell
+xstream > SELECT * FROM my_stream WHERE id > 10;
+[{"...":"..." ....}]
+...
+```
+- Press ``CTRL + C`` to stop the query; 
+
+- If no SQL are type, you can type ``quit`` or ``exit`` to quit the ``xstream`` prompt console.
+

docs/extension/overview.md

@@ -0,0 +1,14 @@
+# Extensions
+
+XStream allows user to customize the different kinds of extensions.  
+
+- The source extension is used for extending different stream source, such as consuming data from other message brokers. XStream has built-in source support for [MQTT broker](../rules/sources/mqtt.md).
+- Sink/Action extension is used for extending pub/push data to different targets, such as database, other message system, web interfaces or file systems. Built-in action support in XStream, see [MQTT](../rules/sinks/mqtt.md) & [log files](../rules/sinks/logs.md).
+- Functions extension allows user to extend different functions that used in SQL. Built-in functions supported in XStream, see [functions](../sqls/built-in_functions.md).
+
+Please read below for how to realize the different extensions.
+
+- [Source extension](#)
+- [Sink/Action extension](#)
+- [Functions extension](#)
+

docs/getting_started.md

@@ -0,0 +1,151 @@
+
+
+## Download & install
+
+TODO
+
+
+
+## Directory structure 
+
+Below is the installation directory structure after installing xstream. 
+
+```
+xstream_installed_dir
+  bin
+    server
+    cli
+  etc
+    mqtt_source.yaml
+    ...
+  data
+    ...
+  plugins
+    ...
+  log
+    ...
+```
+
+## Run the first rule stream
+
+XStream rule is composed by a SQL and multiple actions. XStream SQL is an easy to use SQL-like language to specify the logic of the rule stream. By providing the rule through CLI, a rule stream will be created in the rule engine and run continuously. The user can then manage the rules through CLI.
+
+XStream has a lot of built-in functions and extensions available for complex analysis, and you can find more information about the grammer and its functions from the [XStream SQL reference](sqls/overview.md).
+
+Let's consider a sample scenario where we are receiving temperature and humidity record from a sensor through MQTT service and we want to issue an alert when the temperature is bigger than 30 degrees celcius in a time window. We can write a XStream rule for the above scenario using the following several steps.
+
+### Prerequisite
+
+We assume there is already a MQTT broker as the data source of XStream server. If you don't have one, EMQX is recommended. Please follow the [EMQ Installation Guide](https://docs.emqx.io/broker/v3/en/install.html) to setup a mqtt broker.
+
+### Start the XStream Engine Server
+
+Run bin/server to start the XStream Enginer Server
+```sh
+$ bin/server
+```
+You should see a succesul message `Serving Rule server on port 20498` 
+
+### Defining the input stream
+
+The stream needs to have a name and a schema defining the data that each incoming event should contain. For this scenario, we will use an MQTT source to consume temperature events. The input stream can be defined by SQL language.
+
+We create a stream named demo which consumes mqtt demo topic as specified in the DATASOURCE property.
+```sh
+$ bin/cli create stream demo '(temperature float, humidity bigint) WITH (FORMAT="JSON", DATASOURCE="demo")'
+```
+The mqtt source will connect to mqtt broker at `tcp://localhost:1883`, if your mqtt broker is in another location, specify it in the `etc/mqtt_source.yaml`.  You can change the servers configuration as in below.
+
+```yaml
+default:
+  qos: 1
+  sharedsubscription: true
+  servers: [tcp://127.0.0.1:1883]
+```
+
+You can use command ``cli show streams`` to see if the ``demo`` stream was created or not.
+
+### Testing the stream through query tool
+
+Now the stream is created, it can be tested from ``cli query`` command. The ``xstream`` prompt is displayed as below after typing ``cli query``.
+
+```sh
+$ bin/cli query
+xstream > 
+```
+
+In the ``xstream`` prompt, you can type SQL and validate the SQL against the stream.
+
+```sh
+xstream > select count(*), avg(humidity) as avg_hum, max(humidity) as max_hum from demo where temperature > 30 group by TUMBLINGWINDOW(ss, 5);
+
+query is submit successfully.
+```
+
+Now if any data are publish to the MQTT server available at ``tcp://127.0.0.1:1883``, then it prints message as following.
+
+```
+xstream > [{"avg_hum":41,"count":4,"max_hum":91}]
+[{"avg_hum":62,"count":5,"max_hum":96}]
+[{"avg_hum":36,"count":3,"max_hum":63}]
+[{"avg_hum":48,"count":3,"max_hum":71}]
+[{"avg_hum":40,"count":3,"max_hum":69}]
+[{"avg_hum":44,"count":4,"max_hum":57}]
+[{"avg_hum":42,"count":3,"max_hum":74}]
+[{"avg_hum":53,"count":3,"max_hum":81}]
+...
+```
+
+You can press ``ctrl + c`` to break the query, and server will terminate streaming if detecting client disconnects from the query. Below is the log print at server.
+
+```
+...
+time="2019-09-09T21:46:54+08:00" level=info msg="The client seems no longer fetch the query result, stop the query now."
+time="2019-09-09T21:46:54+08:00" level=info msg="stop the query."
+...
+```
+
+### Writing the rule
+
+As part of the rule, we need to specify the following:
+* rule name: the id of the rule. It must be unique
+* sql: the query to run for the rule
+* actions: the output actions for the rule
+
+We can run the cli rule command to create rule and specify the rule definition in a file
+
+```sh
+$ bin/cli create rule ruleDemo -f myRule
+```
+The content of `myRule` file. It prints out to the log  for the events where the average temperature in a 1 minute tumbling window is bigger than 30.
+```json
+{
+    "sql": "SELECT temperature from demo where temperature > 30",
+    "actions": [{
+        "log":  {}
+    }]
+}
+```
+You should see a succesul message `rule ruleDemo created` in the stream log. And the rule is now up and running.
+
+### Testing the rule
+Now the rule engine is ready to receive events from mqtt demo topic. To test it, just use a mqtt client to publish message to the demo topic. The message should be in json format like this:
+```json
+{"temperature":31.2, "humidity": 77}
+```
+
+Check the stream log located at `log/stream.log`, you would see the filtered data are printed out. Also, if you send below message, it does not meet the SQL condition, and the message will be filtered.
+
+```json
+{"temperature":29, "humidity": 80}
+```
+
+### Managing the rules
+You can use CLI to stop the rule for a while and restart it and other management work. The rule name is the identifier of a rule. Check [Rule Management CLI](cli/rules.md) for detail
+```sh
+$ bin/cli stop rule ruleDemo
+```
+
+
+
+If you'd like to know more about the project, please refer to [doc home](index.md).

docs/index.md

@@ -0,0 +1,12 @@
+
+
+
+
+Refer to the following topics for guidance on using the XStream.
+
+- [Install and operation](operation/overview.md)
+- [Command line interface tools - CLI](cli/overview.md)
+- [XStream SQL reference](sqls/overview.md)
+- [Rules](rules/overview.md)
+- [Extend XStream](extension/overview.md)
+

docs/operation/configuration_file.md

@@ -0,0 +1,11 @@
+# Basic configurations
+The configuration file for XStream is at ``$xstream/etc/xstream.yaml``. The configuration file is yaml format.
+
+## Log level
+
+```yaml
+basic:
+  # true|false, with debug level, it prints more debug info
+  debug: false
+```
+

docs/operation/install/cent-os.md

@@ -0,0 +1,20 @@
+# CentOS
+
+This document describes how to install on CentOS.
+
+## Install from zip
+
+Unzip the installation package.
+
+``unzip xstream-centos7-v0.0.1.zip``
+
+Run the ``cli`` to verify XStream is installed successfully or not.
+
+```shell
+# cd xstream
+# bin/cli --version
+xstream version 0.0.1
+```
+
+If it can print the version, then XStream is installed successfully. 
+

docs/operation/install/overview.md

@@ -0,0 +1,47 @@
+# Installation instruction
+
+Please download the installation package, and refer to below for the instruction of installing for different operate systems.
+
+- [Cent-OS](cent-os.md)
+- ...
+
+# Installation structure 
+
+Below is the directory structure after installation. 
+
+```shell
+bin
+  cli
+etc
+  mqtt_source.yaml
+  *.yaml
+data
+plugins
+log
+```
+
+## bin
+
+The ``bin`` directory includes all of executable files. Such as ``cli`` command.
+
+## etc
+
+The ``etc`` directory contains the configuration files of XStream. Such as MQTT source configurations etc.
+
+## data
+
+XStream persistences all the definitions of streams and rules, and all of message will be stored in this folder  for long duration operations.
+
+## plugins
+
+XStream allows users to develop your own plugins, and put these plugins into this folder.  See [extension](../../extension/overview.md) for more info for how to extend the XStream.
+
+## log
+
+All of the log files are under this folder. The default log file name is ``stream.log``.
+
+# Next steps
+
+- See [getting started](../../getting_started.md) for your first XStream experience.
+- See [CLI tools](../../cli/overview.md) for usage of XStream CLI tools.
+

docs/operation/operations.md

@@ -0,0 +1,12 @@
+# Configuration
+
+- [XStream basic configuration](configuration_file.md)
+- [MQTT source configuration](../rules/sources/mqtt.md)
+
+# Restful APIs
+
+XStream provides some RESTful management APIs.
+
+
+
+

docs/operation/overview.md

@@ -0,0 +1,6 @@
+
+XStream is developed by Golang, and it can be run at different operating systems. See below docs for how to install and operating XStream.
+
+- [Install instruction](install/overview.md)
+- [Operation guide](operations.md)
+

docs/rules/overview.md

@@ -0,0 +1,58 @@
+# Rules 
+
+Rules are defined by JSON, below is an example.
+
+```json
+{
+  "id": "rule1",
+  "sql": "SELECT demo.temperature, demo1.temp FROM demo left join demo1 on demo.timestamp = demo1.timestamp where demo.temperature > demo1.temp GROUP BY demo.temperature, HOPPINGWINDOW(ss, 20, 10)",
+  "actions": [
+    {
+      "log": {}
+    },
+    {
+      "mqtt": {
+        "server": "tcp://47.52.67.87:1883",
+        "topic": "demoSink"
+      }
+    }
+  ]
+}
+```
+
+The following 3 parameters are required for creating a rule.
+
+## Parameters
+
+| Parameter name | Optional | Description                                                  |
+| ------------- | -------- | ------------------------------------------------------------ |
+| id | false   | The id of the rule |
+| sql        | false   | The sql query to run for the rule |
+| actions           | false    | An array of sink actions        |
+| options           | true    | A map of options        |
+
+## id
+
+The identification of the rule. The rule name cannot be duplicated in the same XStream instance.
+
+## sql
+
+The sql query to run for the rule. 
+
+- XStream provides embeded support MQTT source, see  [MQTT source stream](sources/mqtt.md) for more detailed info.
+- See [SQL](../sqls/overview.md) for more info of XStream SQL.
+- Sources can be customized, see [extension](../extension/overview.md) for more detailed info.
+
+### actions
+
+Currently, 2 kinds of actions are supported: [log](sinks/logs.md) and [mqtt](sinks/mqtt.md). Each action can define its own properties.
+
+Actions could be customized to support different kinds of outputs, see [extension](../extension/overview.md) for more detailed info.
+
+### options
+The current options includes:
+
+| Option name | Type & Default Value | Description                                                  |
+| ------------- | -------- | ------------------------------------------------------------ |
+| isEventTime | boolean: false   | Whether to use event time or processing time as the timestamp for an event. If event time is used, the timestamp will be extracted from the payload. The timestamp filed must be specified by the [stream]([extension](../sqls/streams.md)) definition. |
+| lateTolerance        | int64:0   | When working with event-time windowing, it can happen that elements arrive late. LateTolerance can specify by how much time(unit is millisecond) elements can be late before they are dropped. By default, the value is 0 which means late elements are dropped.  |

docs/rules/sinks/logs.md

@@ -0,0 +1,6 @@
+# Log action
+
+The action is used for print output message into log file, the log file is at  `` $xstream_install/log/stream.log`` by default.
+
+No properties can be specified for the action.
+

docs/rules/sinks/mqtt.md

@@ -0,0 +1,10 @@
+# MQTT action
+
+The action is used for publish output message into a MQTT server. 
+
+| Property name | Optional | Description                                                  |
+| ------------- | -------- | ------------------------------------------------------------ |
+| server        | false    | The broker address of the mqtt server, such as ``tcp://127.0.0.1:1883`` |
+| topic         | false    | The mqtt topic, such as ``analysis/result``                  |
+| clientId      | true     | The client id for mqtt connection. If not specified, an uuid will be used |
+

docs/rules/sources/mqtt.md

@@ -0,0 +1,49 @@
+# MQTT source 
+
+XStream provides built-in support for MQTT source stream, which can subscribe the message from MQTT broker and feed into the XStream processing pipeline.  The configuration file of MQTT source is at ``$xstream/etc/mqtt_source.yaml``. Below is the file format.
+
+```yaml
+#Global MQTT configurations
+default:
+  qos: 1
+  sharedsubscription: true
+  servers: [tcp://127.0.0.1:1883]
+  #TODO: Other global configurations
+
+
+#Override the global configurations
+demo: #Conf_key
+  qos: 0
+  servers: [tcp://10.211.55.6:1883]
+```
+
+## Global MQTT configurations
+
+Use can specify the global MQTT settings here. The configuration items specified in ``default`` section will be taken as default settings for all MQTT connections. 
+
+### qos
+
+The default subscription QoS level.
+
+### sharedsubscription
+
+Whether use the shared subscription mode or not. If using the shared subscription mode, then if there are multiple XStream process can be load balanced.
+
+### servers
+
+The server list for MQTT message broker. Currently, only ``ONE`` server can be specified.
+
+## Override the default settings
+
+If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with ``demo``.  Then you can specify the configuration with option ``CONF_KEY`` when creating the stream definition (see [stream specs](../../sqls/streams.md) for more info).
+
+**Sample**
+
+```
+demo (
+		...
+	) WITH (datasource="test/", FORMAT="JSON", KEY="USERID", CONF_KEY="demo");
+```
+
+The configuration keys used for these specific settings are the same as in ``default`` settings, any values specified in specific settings will overwrite the values in ``default`` section.
+

docs/sqls/built-in_functions.md

@@ -0,0 +1,93 @@
+# Functions
+
+XStream has many built-in functions for performing calculations on data.
+
+## Aggregate Functions
+Aggregate functions perform a calculation on a set of values and return a single value. Aggregate functions can be used as expressions only in the following:
+* The select list of a SELECT statement (either a subquery or an outer query).
+* A HAVING clause.
+
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| avg      | avg(col1)   | The average of the values in a group           |
+| count    | count(*)    | The number of items in a group                 |
+| max      | max(col1)   | The maximum value in a group                   |
+| min      | min(col1)   | The minimum value in a group                   |
+| sum      | sum(col1)   | The sum of all the values in a group           |
+
+## Mathematical Functions
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| abs      | abs(col1)   | The absolute value of a value                  |
+| acos     | acos(col1)  | The inverse cosine of a number in radians      |
+| asin     | asin(col1)  | The inverse sine of a number in radians        |
+| atan     | atan(col1)  | The inverse tangent of a number in radians     |
+| atan2    | atan2(col1, col2)  | The angle, in radians,  between the positive x-axis and the (x, y) point defined in the two arguments        |
+| bitand   | bitand(col1, col2)  | Performs a bitwise AND on the bit representations of the two Int(-converted) arguments                     |
+| bitor    | bitor(col1, col2)  | Performs a bitwise OR of the bit representations of the two arguments                                               |
+| bitxor   | bitxor(col1, col2)  | Performs a bitwise XOR on the bit representations of the two Int(-converted) arguments                     |
+| bitnot   | bitnot(col1)| Performs a bitwise NOT on the bit representations of the Int(-converted) argument                                          |
+| ceil     | ceil(col1)  | Round a value up to the nearest BIGINT value.  |
+| cos      | cos(col1)   | Returns the cosine of a number in radians.     |
+| cosh     | cosh(col1)  | Returns the hyperbolic cosine of a number in radians.                                                                 |
+| exp      | exp(col1)   | Returns e raised to the Decimal argument.      |
+| ln       | ln(col1)    | Returns the natural logarithm of the argument. |
+| log      | log(col1)   | Returns the base 10 logarithm of the argument. |
+| mod      | mod(col1, col2)   | Returns the remainder of the division of the first argument by the second argument.                                  |
+| power    | power(x, y) | Pow returns x**y, the base-x exponential of y. |
+| rand     | rand()      | Returns a pseudorandom, uniformly distributed double between 0.0 and 1.0.                                              |
+| round    | round(col1) | Round a value to the nearest BIGINT value.     |
+| sign     | sign(col1)  | Returns the sign of the given number. When the sign of the argument is positive, 1 is returned. When the sign of the argument is negative, -1 is returned. If the argument is 0, 0 is returned.|
+| sin      | sin(col1)   | Returns the sine of a number in radians.       |
+| sinh     | sinh(col1)  | Returns the hyperbolic sine of a number in radians.                                                                 |
+| sqrt     | sqrt(col1)  | Returns the square root of a number.           |
+| tan      | tan(col1)   | Returns the tangent of a number in radians.   |
+| tanh     | tanh(col1)  | Returns the hyperbolic tangent of a number in radians.     |
+
+## String Functions
+
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| concat   | concat(col1...)  | Concatenates arrays or strings. This function accepts any number of arguments and returns a String or an Array        |
+| endswith | endswith(col1, col2) | Returns a Boolean indicating whether the first String argument ends with the second String argument.              |
+| format_time| parse_time(col1, format) | Format a datetime to string.    |
+| indexof  | indexof(col1, col2)  | Returns the first index (0-based) of the second argument as a substring in the first argument.                    |
+| length   | length(col1)| Returns the number of characters in the provided string.                                                                  |
+| lower    | lower(col1) | Returns the lowercase version of the given String.                                                                         |
+| lpad     | lpad(col1, 2) | Returns the String argument, padded on the left side with the number of spaces specified by the second argument.         |
+| ltrim    | ltrim(col1) | Removes all leading whitespace (tabs and spaces) from the provided String.                                                |
+| numbytes | numbytes(col1) | Returns the number of bytes in the UTF-8 encoding of the provided string.                                         | 
+| regexp_matches| regexp_matches(col1, regex) | Returns true if the string (first argument) contains a match for the regular expression.            |
+| regexp_replace| regexp_matches(col1, regex, str) | Replaces all occurrences of the second argument (regular expression) in the first argument with the third argument.                                                          |
+| regexp_substr| regexp_substr(col1, regex) | Finds the first match of the 2nd parameter (regex) in the first parameter.                            |
+| rpad     | rpad(col1, 2) | Returns the String argument, padded on the right side with the number of spaces specified by the second argument.        |
+| rtrim    | rtrim(col1) | Removes all trailing whitespace (tabs and spaces) from the provided String.                                                |
+| substring| substring(col1, start, end) |  returns the substring of the provided String from the provided Int index (0-based, inclusive) to the end of the String.                                                           |
+| startswith| startswith(col1, str) | Returns Boolean, whether the first string argument starts with the second string argument.                  |
+| trim      | trim(col1) | Removes all leading and trailing whitespace (tabs and spaces) from the provided String.                                    |
+| upper     | upper(col1)| Returns the uppercase version of the given String.|
+
+## Conversion Functions
+
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| cast     | cast(col,  "bigint") | Converts a value from one data type to another. The supported types includes: bigint, float, string, boolean and datetime(not supported now). |
+| chr      | chr(col1)   | Returns the ASCII character that corresponds to the given Int argument                                                   |
+| encode   | encode(col1, "base64") |Use the encode function to encode the payload, which potentially might be non-JSON data, into its string representation based on the encoding scheme. Currently, only "base64" econding type is supported.                             |
+| trunc    | trunc(dec, int)| Truncates the first argument to the number of Decimal places specified by the second argument. If the second argument is less than zero, it is set to zero. If the second argument is greater than 34, it is set to 34. Trailing zeroes are stripped from the result.       |
+
+## Hashing Functions
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| md5      | md5(col1)   | Hashed value of the argument                   |
+| sha1     | sha1(col1)  | Hashed value of the argument                   |
+| sha256   | sha256(col1)| Hashed value of the argument                   |
+| sha384   | sha384(col1)| Hashed value of the argument                   |
+| sha512   | sha512(col1)| Hashed value of the argument                   |
+
+## Other Functions
+| Function | Example     | Description                                    |
+| -------- | ----------- | ---------------------------------------------- |
+| isNull   | isNull(col1)| Returns true if the argument is the Null value.|
+| newuuid  | newuuid()   | Returns a random 16-byte UUID.                 |
+| timestamp| timestamp() | Returns the current timestamp in milliseconds from 00:00:00 Coordinated Universal Time (UTC), Thursday, 1 January 1970 |

docs/sqls/data_types.md

@@ -0,0 +1,29 @@
+# Data types
+
+In XStream, each column or an expression has a related data type. A data type describes (and constrains) the set of values that a column of that type can hold or an expression of that type can produce.
+
+
+
+## Supported data types
+
+Below is the list of data types supported.
+
+| #    | Data type | Description                                                  |
+| ---- | --------- | ------------------------------------------------------------ |
+| 1    | bigint    | The int type.                                                |
+| 2    | float     | The float type.                                              |
+| 3    | string    | Text values, comprised of Unicode characters.                |
+| 4    | datetime  | datatime type - *Currently it's NOT supported yet*.          |
+| 5    | boolean   | The boolean type, the value could be ``true`` or ``false``.  |
+| 6    | array     | The array type, can be any types from simple data or struct type (#1 - #5, and #7). |
+| 7    | struct    | The complex type. Set of name/value pairs. Values must be of supported data type. |
+
+
+
+## Type conversions
+
+These are the rules governing *data type conversions*:
+
+- ...
+- 
+

docs/sqls/json_expr.md

@@ -0,0 +1,162 @@
+# JSON Expressions
+
+**Sample data**
+
+```json
+{
+  "name": {"first": "Tom", "last": "Anderson"},
+  "age":37,
+  "children": ["Sara","Alex","Jack"],
+  "fav.movie": "Deer Hunter",
+  "friends": [
+    {"first": "Dale", "last": "Murphy", "age": 44},
+    {"first": "Roger", "last": "Craig", "age": 68},
+    {"first": "Jane", "last": "Murphy", "age": 47}
+  ],
+    "followers": {
+        "Group1": [
+		    {"first": "John", "last": "Shavor", "age": 22},
+		    {"first": "Ken", "last": "Miller", "age": 33}
+        ],
+        "Group2": [
+            {"first": "Alice", "last": "Murphy", "age": 33},
+		    {"first": "Brian", "last": "Craig", "age": 44}
+        ]
+    }
+   "ops": {
+   	"functionA": {"numArgs": 2},
+    "functionB": {"numArgs": 3},
+    "functionC": {"variadic": true}
+  }
+}
+```
+
+## Basic expressions
+
+### Identifier 
+
+Source Dereference (`.`) The source dereference operator can be used to specify columns by dereferencing the source stream or table. The ``->`` dereference selects a key in a nested JSON object.
+
+```
+SELECT demo.age FROM demo
+{"age" : 37}
+```
+
+
+
+```
+SELECT demo.name->first FROM demo
+{"first" : "Tom"}
+```
+
+
+
+```
+SELECT name->first AS fname FROM demo
+{"fname": "Tom"}
+```
+
+### Index expression
+
+Index Expressions allow you to select a specific element in a list. It should look similar to array access in common programming languages. Indexing is 0 based.
+
+```
+SELECT children FROM demo
+
+{
+    "children": ["Sara","Alex","Jack"]
+}
+```
+
+
+
+```
+SELECT children[0] FROM demo
+
+{
+    "children": "Sara"
+}
+
+SELECT d.friends[0]->last FROM demo AS d
+
+{
+    "last" : "Murphy"
+}
+```
+
+### Slicing
+
+Slices allow you to select a contiguous subset of an array. 
+
+``field[from:to]`` If from is not specified, then it means start from the 1st element of array; If to is not specified, then it means end with the last element of array.
+
+```
+SELECT children[0:1] FROM demo
+
+{
+    "children": ["Sara","Alex"]
+}
+```
+
+
+
+```
+SELECT children[:] FROM demo == SELECT children FROM demo
+
+{
+    "children": ["Sara","Alex","Jack"]
+}
+```
+
+
+
+```
+SELECT children[:1] FROM demo
+
+{
+    "children": ["Sara","Alex"]
+}
+```
+
+
+
+```
+SELECT followers->Group1[:1]->first FROM demo
+
+{
+    "first": ["John","Alice"]
+}
+```
+
+
+
+### *Projections* - *NOT SUPPORT YET*
+
+#### List & Slice projections
+
+A wildcard expression creates a list projection, which is a projection over a JSON array. 
+
+```
+SELECT demo.friends[*]->first FROM demo
+{
+    "first": ["Dale", "Roger", "Jane"]
+}
+```
+
+
+
+```
+SELECT friends[:1]->first FROM demo
+{
+    "first": ["Dale", "Roger"]
+}
+```
+
+#### Object projections
+
+```
+SELECT ops->*->numArgs FROM demo
+
+{ "numArgs" : [2, 3] }
+```
+

docs/sqls/overview.md

@@ -0,0 +1,8 @@
+XStream offers a SQL-like query language for performing transformations and computations over streams of events. This document describes the syntax, usage and best practices for the XStream query language. 
+
+- [Stream specifications](streams.md)
+
+- [Query languange element](query_language_elements.md)
+- [Windows](windows.md)
+- [Built-in functions](built-in_functions.md)
+

docs/sqls/query_language_elements.md

@@ -0,0 +1,307 @@
+
+# Query language elements
+
+XStream provides a variety of elements for building queries. They are summarized below.
+
+| Element               | Summary                                                      |
+| --------------------- | ------------------------------------------------------------ |
+| [SELECT](#SELECT)     | SELECT is used to retrieve rows from input streams and enables the selection of one or many columns from one or many input streams in XStream. |
+| [FROM](#FROM)         | FROM specifies the input stream. The FROM clause is always required for any SELECT statement. |
+| [JOIN](#JOIN)         | JOIN is used to combine records from two or more input streams. JOIN includes LEFT, RIGHT, FULL & CROSS. |
+| [WHERE](#WHERE)       | WHERE specifies the search condition for the rows returned by the query. |
+| [GROUP BY](#GROUP BY) | GROUP BY groups a selected set of rows into a set of summary rows grouped by the values of one or more columns or expressions. |
+| [ORDER BY](#ORDER BY) | Order the rows by values of one or more columns.             |
+| [HAVING](#HAVING)     | HAVING specifies a search condition for a group or an aggregate. HAVING can be used only with the SELECT expression.             |
+|                       |                                                              |
+
+
+
+## SELECT
+
+Retrieves rows from input streams and enables the selection of one or many columns from one or many input streams in XStream.
+
+### Syntax
+
+```sql
+SELECT 
+	*
+	| [source_stream.]column_name [AS column_alias]
+	| expression
+  
+```
+
+### Arguments
+
+Specifies that all columns from all input streams in the FROM clause should be returned. The columns are returned by input source, as specified in the FROM clause, and in the order in which they exist in the incoming stream or specified by ORDER BY clause.
+
+**\***
+
+Select all of fields from source stream.
+
+**source_stream**
+
+The source stream name or alias name.
+
+**column_name**
+
+Is the name of a column to return.  If the column to specified is a embedded nest record type, then use the [JSON expressions](json_expr.md) to refer the embedded columns. 
+
+**column_alias**
+
+Is an alternative name to replace the column name in the query result set.  Aliases are used also to specify names for the results of expressions. column_alias cannot be used in a WHERE, GROUP BY, or HAVING clause.
+
+**expression**
+
+Expression is a constant, function, any combination of column names, constants, and functions connected by an operator or operators.
+
+## FROM
+
+Specifies the input stream. The FROM clause is always required for any SELECT statement.
+
+### Syntax
+
+```sql
+FROM source_stream | source_stream AS source_stream_alias 
+```
+
+### Arguments
+
+**source_stream | source_stream_alias**
+
+The input stream name or alias name.
+
+## JOIN
+
+JOIN is used to combine records from two or more input streams. JOIN includes LEFT, RIGHT, FULL & CROSS. 
+
+### Syntax
+
+```sql
+LEFT | RIGHT | FULL | CROSS 
+JOIN 
+source_stream | source_stream AS source_stream_alias
+ON <source_stream|source_stream_alias>.column_name =<source_stream|source_stream_alias>.column_name
+```
+
+### Arguments
+
+**LEFT**
+
+The LEFT JOIN keyword returns all records from the left stream (stream1), and the matched records from the right stream (stream2). The result is NULL from the right side, if there is no match.
+
+```sql
+SELECT column_name(s)
+FROM stream1
+LEFT JOIN stream2
+ON stream1.column_name = stream2.column_name;
+```
+
+**RIGHT**
+
+The RIGHT JOIN keyword returns all records from the right stream (stream2), and the matched records from the left stream (stream1). The result is NULL from the left side, when there is no match.
+
+```sql
+SELECT column_name(s)
+FROM stream1
+RIGHT JOIN stream2
+ON stream1.column_name = stream2.column_name;
+```
+
+**FULL**
+
+The FULL JOIN keyword return all records when there is a match in left (stream1) or right (stream2) table records. 
+
+**Note:** FULL JOIN can potentially return large result-sets!
+
+```sql
+SELECT column_name(s)
+FROM stream1
+FULL JOIN stream2
+ON stream1.column_name = stream2.column_name
+WHERE condition;
+```
+
+**CROSS**
+
+The CROSS JOIN is used to combine each row of the first stream (stream1) with each row of the second stream (stream2). It is also known as the Cartesian join since it returns the Cartesian product of the sets of rows from the joined tables. Let's say if there are **m** rows in stream1, and **n** rows in stream2, then the result of CROSS  JOIN returns **m*n** rows.
+
+**Note:** CROSS JOIN can potentially return very large result-sets!
+
+```sql
+SELECT column_name(s)
+FROM stream1
+CROSS OUTER JOIN stream2
+ON stream1.column_name = stream2.column_name
+WHERE condition;
+```
+
+**source_stream | source_stream_alias**
+
+The input stream name or alias name to be joined.
+
+**column_name**
+
+Is the name of a column to return.  If the column to specified is a embedded nest record type, then use the [JSON expressions](json_expr.md) to refer the embedded columns. 
+
+## WHERE
+
+WHERE specifies the search condition for the rows returned by the query. The WHERE clause is used to extract only those records that fulfill a specified condition.
+
+### Syntax
+
+```
+WHERE <search_condition>
+<search_condition> ::=   
+    { <predicate> | ( <search_condition> ) }   
+    [ { AND | OR } { <predicate> | ( <search_condition> ) } ]   
+[ ,...n ]   
+<predicate> ::=   
+    { expression { = | < > | ! = | > | > = | < | < = } expression   
+```
+
+### Arguments
+
+Expression is a constant, function, any combination of column names, constants, and functions connected by an operator or operators.
+
+**< search_condition >**
+
+Specifies the conditions for the rows returned in the result set for a SELECT statement or query expression. There is no limit to the number of predicates that can be included in a search condition.
+
+**AND**
+
+Combines two conditions and evaluates to TRUE when both of the conditions are TRUE.
+
+**OR**
+
+Combines two conditions and evaluates to TRUE when either condition is TRUE.
+
+**< predicate >**
+
+Is an expression that returns TRUE or FALSE.
+
+**expression**
+
+Is a column name, a constant, a function, a variable, a scalar subquery, or any combination of column names, constants, and functions connected by an operator or operators, or a subquery. The expression can also contain the CASE expression.
+
+**=**
+
+Is the operator used to test the equality between two expressions.
+
+**<>**
+
+Is the operator used to test the condition of two expressions not being equal to each other.
+
+**!=**
+
+Is the operator used to test the condition of two expressions not being equal to each other.
+
+**>**
+
+Is the operator used to test the condition of one expression being greater than the other.
+
+**>=**
+
+Is the operator used to test the condition of one expression being greater than or equal to the other expression.
+
+**<**
+
+Is the operator used to test the condition of one expression being less than the other.
+
+**<=**
+
+Is the operator used to test the condition of one expression being less than or equal to the other expression.
+
+```sql
+SELECT column1, column2, ...
+FROM table_name
+WHERE condition;
+```
+
+
+
+## GROUP BY
+
+GROUP BY groups a selected set of rows into a set of summary rows grouped by the values of one or more columns or expressions.
+
+### Syntax
+
+```sql
+GROUP BY <group by spec>  
+  
+<group by spec> ::=  
+    <group by item> [ ,...n ]  
+    | <window_type>  
+  
+<group by item> ::=  
+    <column_expression>  
+```
+
+## Arguments
+
+**<window_type>**
+
+Specifies any XStream supported Windowing, see [windows](windows.md) for more info.
+
+**< column_expression >**
+
+Is the expression or the name of the column on which the grouping operation is performed. The column expression cannot contain a column alias that is defined in the SELECT list.
+
+```sql
+SELECT column_name(s)
+FROM stream1
+GROUP BY column_name
+```
+
+### HAVING
+
+Specifies a search condition for a group or an aggregate. HAVING can be used only with the SELECT expression. HAVING is typically used in a GROUP BY clause. When GROUP BY is not used, HAVING behaves like a WHERE clause.
+
+#### Syntax
+
+```sql
+[ HAVING <search condition> ]  
+```
+
+#### Arguments
+
+**< search_condition >**
+
+Specifies the search condition for the group or the aggregate to meet.
+
+```sql
+SELECT temp AS t, name FROM topic/sensor1 WHERE name = "dname" GROUP BY name HAVING count(name) > 3
+```
+
+## ORDER BY
+
+Order the rows by values of one or more columns. 
+
+### Syntax
+
+```sql
+ORDER BY column1, column2, ... ASC|DESC
+```
+
+The ORDER BY statement in sql is used to sort the fetched data in either ascending or descending according to one or more columns.
+
+- By default ORDER BY sorts the data in **ascending order.**
+- The keyword DESC is used to sort the data in descending order and the keyword ASC to sort in ascending order.
+
+### Arguments
+
+**ASC**
+
+To sort the data in ascending order.
+
+**DESC**
+
+To sort the data in descending order.
+
+
+
+```sql
+SELECT column1, column2, ...
+FROM table_name
+ORDER BY column1, column2, ... ASC|DESC;
+```
+

docs/sqls/streams.md

@@ -0,0 +1,87 @@
+# Stream specs 
+
+## Data types
+
+In XStream, each column or an expression has a related data type. A data type describes (and constrains) the set of values that a column of that type can hold or an expression of that type can produce.
+
+Below is the list of data types supported.
+
+| #    | Data type | Description                                                  |
+| ---- | --------- | ------------------------------------------------------------ |
+| 1    | bigint    |                                                              |
+| 2    | float     |                                                              |
+| 3    | string    |                                                              |
+| 4    | datetime  | Not support.                                                 |
+| 5    | boolean   |                                                              |
+| 6    | array     | The array type, can be any simple types or struct type (#1 - #5, and #7). |
+| 7    | struct    | The complex type.                                            |
+
+## Language definitions
+
+```sql
+CREATE STREAM   
+    stream_name   
+    ( column_name <data_type> [ ,...n ] )
+    WITH ( property_name = expression [, ...] );
+```
+
+**The supported property names.**
+
+| Property name | Optional | Description                                                  |
+| ------------- | -------- | ------------------------------------------------------------ |
+| DATASOURCE | false    | The topic names list if it's a MQTT data source. |
+| FORMAT        | false    | JSON. |
+| KEY           | true     | Reserved key, currently the field is not used. It will be used for GROUP BY statements. |
+| TYPE     | false    | The data format, currently the value can only be "JSON". |
+| StrictValidation     | false    | To control validation behavior of message field against stream schema. See [StrictValidation](#StrictValidation) for more info. |
+| CONF_KEY | false | If additional configuration items are requied to be configured, then specify the config key here. See [MQTT stream](../rules/sources/mqtt.md) for more info. |
+
+**Example 1,**
+
+```sql
+my_stream 
+  (id bigint, name string, score float)
+WITH ( datasource = "topic/temperature", FORMAT = "json", KEY = "id");
+```
+
+The stream will subscribe to MQTT topic ``topic/temperature``, the server connection uses ``servers`` key of ``default`` section in configuration file ``$xstream/etc/mqtt_source.yaml``. 
+
+- See [MQTT source](../rules/sources/mqtt.md) for more info.
+
+**Example 2,**
+
+```sql
+demo (
+		USERID BIGINT,
+		FIRST_NAME STRING,
+		LAST_NAME STRING,
+		NICKNAMES ARRAY(STRING),
+		Gender BOOLEAN,
+		ADDRESS STRUCT(STREET_NAME STRING, NUMBER BIGINT),
+	) WITH (datasource="test/", FORMAT="JSON", KEY="USERID", CONF_KEY="demo");
+```
+
+The stream will subscribe to MQTT topic ``test/``, the server connection uses settings of ``demo`` section in configuration file ``$xstream/etc/mqtt_source.yaml``. 
+
+- See [MQTT source](../rules/sources/mqtt.md) for more info.
+
+- See [rules and streams CLI docs](../cli/overview.md) for more information of rules & streams management.
+
+### StrictValidation
+
+```
+The value of StrictValidation can be true or false.
+1) True: Drop the message if the message  is not satisfy with the stream definition.
+2) False: Keep the message, but fill the missing field with default empty value.
+
+bigint: 0
+float: 0.0
+string: ""
+datetime: (NOT support yet)
+boolean: false
+array: zero length array
+struct: null value
+```
+
+See [Query languange element](query_language_elements.md) for more inforamtion of SQL language.
+

docs/sqls/windows.md

@@ -0,0 +1,97 @@
+# Windows
+
+In time-streaming scenarios, performing operations on the data contained in temporal windows is a common pattern. XStream has native support for windowing functions, enabling you to author complex stream processing jobs with minimal effort.
+
+There are four kinds of windows to use: [Tumbling window](#TUMBLING WINDOW), [Hopping window](#Hopping window), [Sliding window](#Sliding window), and [Session window](#Session window). You use the window functions in the GROUP BY clause of the query syntax in your XStream queries. 
+
+All the windowing operations output results at the end of the window. The output of the window will be single event based on the aggregate function used. 
+
+## Time-units
+
+There are 5 time-units can be used in the windows. For example, ``TUMBLINGWINDOW(ss, 10)``, which means group the data with tumbling with with 10  seconds interval.
+
+**DD**: day unit
+
+**HH**: hour unit
+
+**MI**: minute unit
+
+**SS**: second unit
+
+**MS**: milli-second unit
+
+## Tumbling window
+
+Tumbling window functions are used to segment a data stream into distinct time segments and perform a function against them, such as the example below. The key differentiators of a Tumbling window are that they repeat, do not overlap, and an event cannot belong to more than one tumbling window.
+
+![Tumbling Window](resources/tumblingWindow.png)
+
+TODO: 
+
+- TIMESTAMP BY is required?
+- Count function is not supported.21
+
+
+
+```sql
+SELECT count(*) FROM demo GROUP BY ID, TUMBLINGWINDOW(ss, 10);
+```
+
+## Hopping window
+
+Hopping window functions hop forward in time by a fixed period. It may be easy to think of them as Tumbling windows that can overlap, so events can belong to more than one Hopping window result set. To make a Hopping window the same as a Tumbling window, specify the hop size to be the same as the window size.
+
+![Hopping Window](resources/hoppingWindow.png)
+
+TODO: 
+
+- TIMESTAMP BY is required?
+- Count function is not supported.
+
+
+
+```sql
+SELECT count(*) FROM demo GROUP BY ID, HOPPINGWINDOW(ss, 10, 5);
+```
+
+
+
+## Sliding window
+
+Sliding window functions, unlike Tumbling or Hopping windows, produce an output **ONLY** when an event occurs. Every window will have at least one event and the window continuously moves forward by an € (epsilon). Like hopping windows, events can belong to more than one sliding window.
+
+![Sliding Window](resources/slidingWindow.png)
+
+TODO: 
+
+- TIMESTAMP BY is required?
+- Count function is not supported.
+
+```sql
+SELECT count(*) FROM demo GROUP BY ID, SLIDINGWINDOW(mm, 1);
+```
+
+
+
+## Session window
+
+Session window functions group events that arrive at similar times, filtering out periods of time where there is no data. It has two main parameters: timeout and maximum duration.
+
+![Session Window](resources/sessionWindow.png)
+
+TODO: 
+
+- TIMESTAMP BY is required?
+- Count function is not supported.
+
+
+
+```sql
+SELECT count(*) FROM demo GROUP BY ID, SESSIONWINDOW(mm, 2, 1);
+```
+
+
+
+A session window begins when the first event occurs. If another event occurs within the specified timeout from the last ingested event, then the window extends to include the new event. Otherwise if no events occur within the timeout, then the window is closed at the timeout.
+
+If events keep occurring within the specified timeout, the session window will keep extending until maximum duration is reached. The maximum duration checking intervals are set to be the same size as the specified max duration. For example, if the max duration is 10, then the checks on if the window exceed maximum duration will happen at t = 0, 10, 20, 30, etc.

etc/client.yaml

@@ -0,0 +1,3 @@
+basic:
+  host: 127.0.0.1
+  port: 20498

etc/mqtt_source.yaml

@@ -0,0 +1,12 @@
+#Global MQTT configurations
+default:
+  qos: 1
+  sharedsubscription: true
+  servers: [tcp://10.211.55.6:1887]
+  #TODO: Other global configurations
+
+
+#Override the global configurations
+demo_conf: #Conf_key
+  qos: 0
+  servers: [tcp://10.211.55.6:1883, tcp://127.0.0.1]

etc/xstream.yaml

@@ -0,0 +1,4 @@
+basic:
+  # true|false, with debug level, it prints more debug info
+  debug: false
+  port: 20498

go.mod

@@ -0,0 +1,17 @@
+module engine
+
+require (
+	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
+	github.com/dgraph-io/badger v2.0.0-rc2+incompatible
+	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/eclipse/paho.mqtt.golang v1.2.0
+	github.com/go-yaml/yaml v2.1.0+incompatible
+	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/google/uuid v1.1.1
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/sirupsen/logrus v1.4.2
+	github.com/urfave/cli v1.22.0
+	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b // indirect
+)

xsql/ast.go

@@ -3,9 +3,10 @@ package xsql
 import (
 	"engine/common"
 	"fmt"
-	"log"
 	"math"
+	"sort"
 	"strings"
+	"time"
 )
 
 
@@ -47,6 +48,9 @@ type JoinType int
 const (
 	LEFT_JOIN JoinType = iota
 	INNER_JOIN
+	RIGHT_JOIN
+	FULL_JOIN
+	CROSS_JOIN
 )
 
 type Join struct {
@@ -68,11 +72,12 @@ type Statement interface{
 }
 
 type SelectStatement struct {
-	Fields    Fields
-	Sources Sources
-	Joins Joins
-	Condition Expr
+	Fields     Fields
+	Sources    Sources
+	Joins      Joins
+	Condition  Expr
 	Dimensions Dimensions
+	Having	   Expr
 	SortFields SortFields
 }
 
@@ -187,6 +192,23 @@ func (d *Dimension) expr() {}
 func (d *Dimension) node(){}
 
 func (d Dimensions) node(){}
+func (d *Dimensions) GetWindow() *Window{
+	for _, child := range *d {
+		if w, ok := child.Expr.(*Window); ok{
+			return w
+		}
+	}
+	return nil
+}
+func (d *Dimensions) GetGroups() Dimensions{
+	var nd Dimensions
+	for _, child := range *d {
+		if _, ok := child.Expr.(*Window); !ok{
+			nd = append(nd, child)
+		}
+	}
+	return nd
+}
 
 func (sf *SortField) expr() {}
 func (sf *SortField) node(){}
@@ -212,14 +234,15 @@ const (
 	SESSION_WINDOW
 )
 
-type Windows struct {
-	Args []Expr
+type Window struct {
 	WindowType WindowType
+	Length	   *IntegerLiteral
+	Interval   *IntegerLiteral
 }
 
-func (w *Windows) expr() {}
-func (w *Windows) literal() {}
-func (w *Windows) node(){}
+func (w *Window) expr()    {}
+func (w *Window) literal() {}
+func (w *Window) node()    {}
 
 type  SelectStatements []SelectStatement
 
@@ -349,10 +372,9 @@ func Walk(v Visitor, node Node) {
 			Walk(v, expr)
 		}
 
-	case *Windows:
-		for _, expr := range n.Args {
-			Walk(v, expr)
-		}
+	case *Window:
+		Walk(v, n.Length)
+		Walk(v, n.Interval)
 
 	case *Field:
 		Walk(v, n.Expr)
@@ -387,6 +409,8 @@ func Walk(v Visitor, node Node) {
 		for _, s := range n {
 			Walk(v, &s)
 		}
+	case *Join:
+		Walk(v, n.Expr)
 
 	case *StreamStmt:
 		Walk(v, &n.Name)
@@ -417,12 +441,23 @@ func Walk(v Visitor, node Node) {
 }
 
 
+// WalkFunc traverses a node hierarchy in depth-first order.
+func WalkFunc(node Node, fn func(Node)) {
+	Walk(walkFuncVisitor(fn), node)
+}
+
+type walkFuncVisitor func(Node)
+
+func (fn walkFuncVisitor) Visit(n Node) Visitor { fn(n); return fn }
+
+
 // Valuer is the interface that wraps the Value() method.
 type Valuer interface {
 	// Value returns the value and existence flag for a given key.
 	Value(key string) (interface{}, bool)
 }
 
+
 // CallValuer implements the Call method for evaluating function calls.
 type CallValuer interface {
 	Valuer
@@ -431,11 +466,53 @@ type CallValuer interface {
 	Call(name string, args []interface{}) (interface{}, bool)
 }
 
-// MapValuer is a valuer that substitutes values for the mapped interface.
-type MapValuer map[string]interface{}
+type AggregateCallValuer interface {
+	CallValuer
+	GetAllTuples() AggregateData
+}
+
+type Wildcarder interface {
+	// Value returns the value and existence flag for a given key.
+	All(stream string) (interface{}, bool)
+}
+
+type DataValuer interface {
+	Valuer
+	Wildcarder
+}
+
+type WildcardValuer struct {
+	Data Wildcarder
+}
+
+//TODO deal with wildcard of a stream, e.g. SELECT Table.* from Table inner join Table1
+func (wv *WildcardValuer) Value(key string) (interface{}, bool) {
+	if key == ""{
+		return wv.Data.All(key)
+	}else{
+		a := strings.Index(key, ".*")
+		if a <= 0{
+			return nil, false
+		}else{
+			return wv.Data.All(key[:a])
+		}
+	}
+}
 
-// Value returns the value for a key in the MapValuer.
-func (m MapValuer) Value(key string) (interface{}, bool) {
+/**********************************
+**	Various Data Types for SQL transformation
+ */
+
+type AggregateData interface {
+	AggregateEval(expr Expr) []interface{}
+}
+
+// Message is a valuer that substitutes values for the mapped interface.
+type Message map[string]interface{}
+
+// Value returns the value for a key in the Message.
+func (m Message) Value(key string) (interface{}, bool) {
+	key = strings.ToLower(key)
 	if keys := strings.Split(key, "."); len(keys) == 1 {
 		v, ok := m[key]
 		return v, ok
@@ -447,138 +524,275 @@ func (m MapValuer) Value(key string) (interface{}, bool) {
 	return nil, false
 }
 
-type WildcardValuer struct {
-	Data map[string]interface{}
+type Event interface {
+	GetTimestamp() int64
+	IsWatermark() bool
 }
 
-func (wv *WildcardValuer) Value(key string) (interface{}, bool) {
-	//TODO Need to read the schema from stream, and fill into the map
-	return wv.Data, true
+type Tuple struct {
+	Emitter   string
+	Message   Message
+	Timestamp int64
 }
 
-
-type EmitterTuple struct {
-	Emitter string
-	Message map[string]interface{}
+func (t *Tuple) Value(key string) (interface{}, bool) {
+	return t.Message.Value(key)
 }
 
-type MergedEmitterTuple struct {
-	MergedMessage []EmitterTuple
+func (t *Tuple) All(stream string) (interface{}, bool) {
+	return t.Message, true
 }
 
-func (me *MergedEmitterTuple) AddMergedMessage(message EmitterTuple) {
-	me.MergedMessage = append(me.MergedMessage, message)
+func (t *Tuple) AggregateEval(expr Expr) []interface{} {
+	return []interface{}{Eval(expr, t)}
 }
 
-type MergedEmitterTupleSets []MergedEmitterTuple
-
-
-type Messages []map[string]interface{}
-
-type EmitterTuples struct {
-	Emitter string
-	Messages Messages
+func (t *Tuple) GetTimestamp() int64 {
+	return t.Timestamp
 }
 
-type EvalResultAndMessage struct {
-	Stream string
-	Result interface{}
-	Message map[string]interface{}
+func (t *Tuple) IsWatermark() bool {
+	return false
 }
 
-type ResultsAndMessages []EvalResultAndMessage
+type WindowTuples struct {
+	Emitter string
+	Tuples []Tuple
+}
 
-type MultiEmitterTuples []EmitterTuples
+type WindowTuplesSet []WindowTuples
 
-func (met *MultiEmitterTuples) GetBySrc(src string) Messages {
-	for _, me := range *met {
+func (w WindowTuplesSet) GetBySrc(src string) []Tuple {
+	for _, me := range w {
 		if me.Emitter == src {
-			return me.Messages
+			return me.Tuples
 		}
 	}
 	return nil
 }
 
-type Tuple struct {
-	EmitterName string
-	Message interface{}
-	Timestamp int64
+func (w WindowTuplesSet) Len() int {
+	if len(w) > 0{
+		return len(w[0].Tuples)
+	}
+	return 0
 }
-
-func (met *MultiEmitterTuples) addTuple(tuple *Tuple) {
-	found := false
-	m, ok := tuple.Message.(map[string]interface{})
-	if !ok {
-		log.Printf("Expect map[string]interface{} for the message type.")
-		return
+func (w WindowTuplesSet) Swap(i, j int) {
+	if len(w) > 0{
+		s := w[0].Tuples
+		s[i], s[j] = s[j], s[i]
+	}
+}
+func (w WindowTuplesSet) Index(i int) Valuer {
+	if len(w) > 0{
+		s := w[0].Tuples
+		return &(s[i])
 	}
+	return nil
+}
 
-	for _, t := range *met {
-		if t.Emitter == tuple.EmitterName {
-			t.Messages = append(t.Messages, m)
+func (w WindowTuplesSet) AddTuple(tuple *Tuple) WindowTuplesSet{
+	found := false
+	for i, t := range w {
+		if t.Emitter == tuple.Emitter {
+			t.Tuples = append(t.Tuples, *tuple)
+			found = true
+			w[i] = t
 			break
 		}
 	}
 
 	if !found {
-		ets := &EmitterTuples{Emitter:tuple.EmitterName}
-		ets.Messages = append(ets.Messages, m)
-		*met = append(*met, *ets)
+		ets := &WindowTuples{Emitter: tuple.Emitter}
+		ets.Tuples = append(ets.Tuples, *tuple)
+		w = append(w, *ets)
+	}
+	return w
+}
+
+//Sort by tuple timestamp
+func (w WindowTuplesSet) Sort() {
+	for _, t := range w {
+		tuples := t.Tuples
+		sort.SliceStable(tuples, func(i, j int) bool {
+			return tuples[i].Timestamp < tuples[j].Timestamp
+		})
+		t.Tuples = tuples
 	}
 }
 
-func (met MultiEmitterTuples) Value(key string) (interface{}, bool) {
-	var ret ResultsAndMessages
-	if keys := strings.Split(key, "."); len(keys) != 2 {
+func (w WindowTuplesSet) AggregateEval(expr Expr) []interface{} {
+	var result []interface{}
+	if len(w) != 1 { //should never happen
+		return nil
+	}
+	for _, t := range w[0].Tuples {
+		result = append(result, Eval(expr, &t))
+	}
+	return result
+}
+
+type JoinTuple struct {
+	Tuples []Tuple
+}
+
+func (jt *JoinTuple) AddTuple(tuple Tuple) {
+	jt.Tuples = append(jt.Tuples, tuple)
+}
+
+func (jt *JoinTuple) AddTuples(tuples []Tuple) {
+	for _, t := range tuples {
+		jt.Tuples = append(jt.Tuples, t)
+	}
+}
+
+func (jt *JoinTuple) Value(key string) (interface{}, bool) {
+	keys := strings.Split(key, ".")
+	tuples := jt.Tuples
+	switch len(keys) {
+	case 1:
+		if len(tuples) > 1 {
+			for _, tuple := range tuples {	//TODO support key without modifier?
+				v, ok := tuple.Message[key]
+				if ok{
+					return v, ok
+				}
+			}
+			common.Log.Infoln("Wrong key: ", key, ", not found")
+			return nil, false
+		} else{
+			v, ok := tuples[0].Message[key]
+			return v, ok
+		}
+	case 2:
+		emitter, key := keys[0], keys[1]
+		//TODO should use hash here
+		for _, tuple := range tuples {
+			if tuple.Emitter == emitter {
+				v, ok := tuple.Message[key]
+				return v, ok
+			}
+		}
+		return nil, false
+	default:
 		common.Log.Infoln("Wrong key: ", key, ", expect dot in the expression.")
 		return nil, false
-	} else {
-		emitter, key := keys[0], keys[1]
-		for _, me := range met {
-			if me.Emitter == emitter {
-				for _, m := range me.Messages {
-					if r, ok := m[key]; ok {
-						rm := &EvalResultAndMessage{Stream: keys[0], Result:r, Message:m}
-						ret = append(ret, *rm)
-					}
+	}
+}
+
+func (jt *JoinTuple) All(stream string) (interface{}, bool) {
+	if stream != ""{
+		for _, t := range jt.Tuples{
+			if t.Emitter == stream{
+				return t.Message, true
+			}
+		}
+	}else{
+		var r Message = make(map[string]interface{})
+		for _, t := range jt.Tuples{
+			for k, v := range t.Message{
+				if _, ok := r[k]; !ok{
+					r[k] = v
 				}
-				break
 			}
 		}
+		return r, true
 	}
+	return nil, false
+}
 
-	if len(ret) > 0 {
-		return ret, true
-	} else {
-		return nil, false
+type JoinTupleSets []JoinTuple
+func (s JoinTupleSets) Len() int      { return len(s) }
+func (s JoinTupleSets) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s JoinTupleSets) Index(i int) Valuer { return &(s[i]) }
+
+func (s JoinTupleSets) AggregateEval(expr Expr) []interface{} {
+	var result []interface{}
+	for _, t := range s {
+		result = append(result, Eval(expr, &t))
 	}
+	return result
 }
 
-func (met *MultiEmitterTuples) AddTuple(tuple *Tuple) {
-	found := false
-	m, ok := tuple.Message.(map[string]interface{})
-	if !ok {
-		common.Log.Printf("Expect map[string]interface{} for the message type.")
-		return
+type GroupedTuples []DataValuer
+func (s GroupedTuples) AggregateEval(expr Expr) []interface{} {
+	var result []interface{}
+	for _, t := range s {
+		result = append(result, Eval(expr, t))
 	}
+	return result
+}
 
-	for _, t := range *met {
-		if t.Emitter == tuple.EmitterName {
-			t.Messages = append(t.Messages, m)
-			break
-		}
+type GroupedTuplesSet []GroupedTuples
+func (s GroupedTuplesSet) Len() int           { return len(s) }
+func (s GroupedTuplesSet) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s GroupedTuplesSet) Index(i int) Valuer { return s[i][0] }
+
+type SortingData interface {
+	Len() int
+	Swap(i, j int)
+	Index(i int) Valuer
+}
+
+// multiSorter implements the Sort interface, sorting the changes within.Hi
+type MultiSorter struct {
+	SortingData
+	fields SortFields
+}
+
+// OrderedBy returns a Sorter that sorts using the less functions, in order.
+// Call its Sort method to sort the data.
+func OrderedBy(fields SortFields) *MultiSorter {
+	return &MultiSorter{
+		fields: fields,
 	}
+}
 
-	if !found {
-		ets := &EmitterTuples{Emitter:tuple.EmitterName}
-		ets.Messages = append(ets.Messages, m)
-		*met = append(*met, *ets)
+// Less is part of sort.Interface. It is implemented by looping along the
+// less functions until it finds a comparison that discriminates between
+// the two items (one is less than the other). Note that it can call the
+// less functions twice per call. We could change the functions to return
+// -1, 0, 1 and reduce the number of calls for greater efficiency: an
+// exercise for the reader.
+func (ms *MultiSorter) Less(i, j int) bool {
+	p, q := ms.SortingData.Index(i), ms.SortingData.Index(j)
+	vep, veq := &ValuerEval{Valuer: MultiValuer(p, &FunctionValuer{})}, &ValuerEval{Valuer: MultiValuer(q, &FunctionValuer{})}
+	for _, field := range ms.fields {
+		vp, ok := vep.Valuer.Value(field.Name)
+		if !ok {
+			return !field.Ascending
+		}
+		vq, ok := veq.Valuer.Value(field.Name)
+		if !ok {
+			return !field.Ascending
+		}
+		switch {
+		case vep.simpleDataEval(vp, vq, LT):
+			return field.Ascending
+		case veq.simpleDataEval(vq, vp, LT):
+			return !field.Ascending
+		}
 	}
+	return false
+}
+
+// Sort sorts the argument slice according to the less functions passed to OrderedBy.
+func (ms *MultiSorter) Sort(data SortingData) {
+	ms.SortingData = data
+	sort.Sort(ms)
+}
+
+type EvalResultMessage struct {
+	Emitter string
+	Result  interface{}
+	Message Message
 }
 
+type ResultsAndMessages []EvalResultMessage
+
 // Eval evaluates expr against a map.
-func Eval(expr Expr, m map[string]interface{}) interface{} {
-	eval := ValuerEval{Valuer: MapValuer(m)}
+func Eval(expr Expr, m Valuer) interface{} {
+	eval := ValuerEval{Valuer: MultiValuer(m, &FunctionValuer{})}
 	return eval.Eval(expr)
 }
 
@@ -589,7 +803,6 @@ type ValuerEval struct {
 	// IntegerFloatDivision will set the eval system to treat
 	// a division between two integers as a floating point division.
 	IntegerFloatDivision bool
-	JoinType JoinType
 }
 
 // MultiValuer returns a Valuer that iterates over multiple Valuer instances
@@ -614,12 +827,66 @@ func (a multiValuer) Call(name string, args []interface{}) (interface{}, bool) {
 		if valuer, ok := valuer.(CallValuer); ok {
 			if v, ok := valuer.Call(name, args); ok {
 				return v, true
+			} else {
+				common.Log.Println(fmt.Sprintf("Found error \"%s\" when call func %s.\n", v, name))
+			}
+		}
+	}
+	return nil, false
+}
+
+type multiAggregateValuer struct{
+	data AggregateData
+	valuers []Valuer
+}
+
+func MultiAggregateValuer(data AggregateData, valuers ...Valuer) Valuer {
+	return &multiAggregateValuer{
+		data: data,
+		valuers: valuers,
+	}
+}
+
+func (a *multiAggregateValuer) Value(key string) (interface{}, bool) {
+	for _, valuer := range a.valuers {
+		if v, ok := valuer.Value(key); ok {
+			return v, true
+		}
+	}
+	return nil, false
+}
+
+//The args is [][] for aggregation
+func (a *multiAggregateValuer) Call(name string, args []interface{}) (interface{}, bool) {
+	var singleArgs []interface{} = nil
+	for _, valuer := range a.valuers {
+		if a, ok := valuer.(AggregateCallValuer); ok {
+			if v, ok := a.Call(name, args); ok {
+				return v, true
+			}
+		}else if c, ok := valuer.(CallValuer); ok{
+			if singleArgs == nil{
+				for _, arg := range args{
+					if arg, ok := arg.([]interface{}); ok{
+						singleArgs = append(singleArgs, arg[0])
+					}else{
+						common.Log.Infof("multiAggregateValuer does not get [][] args but get %v", args)
+						return nil, false
+					}
+				}
+			}
+			if v, ok := c.Call(name, singleArgs); ok {
+				return v, true
 			}
 		}
 	}
 	return nil, false
 }
 
+func (a *multiAggregateValuer) GetAllTuples() AggregateData {
+	return a.data
+}
+
 type BracketEvalResult struct {
 	Start, End int
 }
@@ -646,6 +913,8 @@ func (v *ValuerEval) Eval(expr Expr) interface{} {
 		return v.Eval(expr.Expr)
 	case *StringLiteral:
 		return expr.Val
+	case *BooleanLiteral:
+		return expr.Val
 	case *ColonExpr:
 		return &BracketEvalResult{Start:expr.Start, End:expr.End}
 	case *IndexExpr:
@@ -653,10 +922,17 @@ func (v *ValuerEval) Eval(expr Expr) interface{} {
 	case *Call:
 		if valuer, ok := v.Valuer.(CallValuer); ok {
 			var args []interface{}
+
 			if len(expr.Args) > 0 {
 				args = make([]interface{}, len(expr.Args))
-				for i := range expr.Args {
-					args[i] = v.Eval(expr.Args[i])
+				if aggreValuer, ok := valuer.(AggregateCallValuer); ok{
+					for i := range expr.Args {
+						args[i] = aggreValuer.GetAllTuples().AggregateEval(expr.Args[i])
+					}
+				}else{
+					for i := range expr.Args {
+						args[i] = v.Eval(expr.Args[i])
+					}
 				}
 			}
 			val, _ := valuer.Call(expr.Name, args)
@@ -685,8 +961,6 @@ func (v *ValuerEval) Eval(expr Expr) interface{} {
 func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} {
 	lhs := v.Eval(expr.LHS)
 	switch val := lhs.(type) {
-	case ResultsAndMessages:
-		return v.evalSet(val, expr)
 	case map[string]interface{}:
 		return v.evalJsonExpr(val, expr.OP, expr.RHS)
 	case []interface{}:
@@ -706,8 +980,6 @@ func (v *ValuerEval) evalBinaryExpr(expr *BinaryExpr) interface{} {
 			rhs = false
 		}
 	}
-
-
 	return v.simpleDataEval(lhs, rhs, expr.OP)
 }
 
@@ -717,7 +989,7 @@ func (v *ValuerEval) evalJsonExpr(result interface{}, op Token,  expr Expr) inte
 		switch op {
 		case ARROW:
 			if exp, ok := expr.(*FieldRef); ok {
-				ve := &ValuerEval{Valuer: MapValuer(val)}
+				ve := &ValuerEval{Valuer: Message(val)}
 				return ve.Eval(exp)
 			} else {
 				fmt.Printf("The right expression is not a field reference node.\n")
@@ -764,59 +1036,6 @@ func (v *ValuerEval) evalJsonExpr(result interface{}, op Token,  expr Expr) inte
 	return nil
 }
 
-func (v *ValuerEval) evalSet(lefts ResultsAndMessages, expr *BinaryExpr) interface{} {
-	//For the JSON expressions
-	if expr.OP == ARROW || expr.OP == SUBSET {
-		for i, left := range lefts {
-			lefts[i].Result = v.evalJsonExpr(left.Result, expr.OP, expr.RHS)
-		}
-		return lefts
-	}
-
-	//For the simple type expressions
-	rhs := v.Eval(expr.RHS)
-	rights, ok := rhs.(ResultsAndMessages)
-	if rhs != nil && !ok {
-		for i, left := range lefts {
-			r := v.simpleDataEval(left.Result, rhs, expr.OP)
-			lefts[i].Result = r
-		}
-		return lefts
-	}
-
-	sets := MergedEmitterTupleSets{}
-	for _, left := range lefts {
-		merged := &MergedEmitterTuple{}
-		lm := &EmitterTuple{string(left.Stream), left.Message}
-		if v.JoinType == LEFT_JOIN {
-			merged.AddMergedMessage(*lm)
-		}
-
-		innerAppend := false
-		for _, right := range rights {
-			r := v.simpleDataEval(left.Result, right.Result, expr.OP)
-			if v1, ok := r.(bool); ok {
-				if v1 {
-					if v.JoinType == INNER_JOIN && !innerAppend{
-						merged.AddMergedMessage(*lm)
-						innerAppend = true
-					}
-					rm := &EmitterTuple{string(right.Stream), right.Message}
-					merged.AddMergedMessage(*rm)
-				}
-			} else {
-				common.Log.Infoln("Evaluation error for set.")
-			}
-		}
-		if len(merged.MergedMessage) > 0 {
-			sets = append(sets, *merged)
-		}
-	}
-
-	return sets
-}
-
-
 func (v *ValuerEval) simpleDataEval(lhs, rhs interface{}, op Token) interface{} {
 	lhs = convertNum(lhs)
 	rhs = convertNum(rhs)
@@ -1153,6 +1372,49 @@ func (v *ValuerEval) simpleDataEval(lhs, rhs interface{}, op Token) interface{}
 				return false
 			}
 			return lhs != rhs
+		case LT:
+			rhs, ok := rhs.(string)
+			if !ok {
+				return false
+			}
+			return lhs < rhs
+		case LTE:
+			rhs, ok := rhs.(string)
+			if !ok {
+				return false
+			}
+			return lhs <= rhs
+		case GT:
+			rhs, ok := rhs.(string)
+			if !ok {
+				return false
+			}
+			return lhs > rhs
+		case GTE:
+			rhs, ok := rhs.(string)
+			if !ok {
+				return false
+			}
+			return lhs >= rhs
+		}
+	case time.Time:
+		rt, err := common.InterfaceToTime(rhs, "")
+		if err != nil{
+			return false
+		}
+		switch op {
+		case EQ:
+			return lhs.Equal(rt)
+		case NEQ:
+			return !lhs.Equal(rt)
+		case LT:
+			return lhs.Before(rt)
+		case LTE:
+			return lhs.Before(rt) || lhs.Equal(rt)
+		case GT:
+			return lhs.After(rt)
+		case GTE:
+			return lhs.After(rt) || lhs.Equal(rt)
 		}
 	}
 
@@ -1222,4 +1484,24 @@ func toFloat64(para interface{}) float64 {
 		return v
 	}
 	return 0
+}
+
+func IsAggStatement(node Node) (bool) {
+	var r bool = false
+	WalkFunc(node, func(n Node) {
+		if f, ok := n.(*Call); ok {
+			fn := strings.ToLower(f.Name)
+			if _, ok1 := aggFuncMap[fn]; ok1 {
+				r = true
+				return
+			}
+		} else if d, ok := n.(Dimensions); ok {
+			ds := d.GetGroups()
+			if ds != nil && len(ds) > 0 {
+				r = true
+				return
+			}
+		}
+	});
+	return r
 }

xsql/ast_agg_stmt_test.go

@@ -0,0 +1,42 @@
+package xsql
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestIsAggStatement(t *testing.T) {
+	var tests = []struct {
+		s    string
+		agg  bool
+		err  string
+	}{
+		{s: `SELECT avg(1) FROM tbl`,agg: true},
+		{s: `SELECT test(1) FROM tbl`,agg: false},
+		{s: `SELECT test(avg(f1)) FROM tbl`,agg: true},
+
+		{s: `SELECT sum(f1) FROM tbl GROUP by f1`,agg: true},
+		{s: `SELECT f1 FROM tbl GROUP by f1`,agg: true},
+
+		{s: `SELECT count(f1) FROM tbl`,agg: true},
+		{s: `SELECT max(f1) FROM tbl`,agg: true},
+		{s: `SELECT min(f1) FROM tbl`,agg: true},
+		{s: `SELECT count(f1) FROM tbl group by tumblingwindow(ss, 5)`,agg: true},
+
+		{s: `SELECT f1 FROM tbl left join tbl2 on tbl1.f1 = tbl2.f2`,agg: false},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		//fmt.Printf("Parsing SQL %q.\n", tt.s)
+		stmt, err := NewParser(strings.NewReader(tt.s)).Parse()
+		isAgg := IsAggStatement(stmt)
+		if !reflect.DeepEqual(tt.err, errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
+		} else if tt.err == "" && (tt.agg != isAgg) {
+			t.Errorf("Error: expected %t, actual %t.", tt.agg, isAgg)
+		}
+	}
+}

xsql/funcs_aggregate.go

@@ -0,0 +1,188 @@
+package xsql
+
+import (
+	"fmt"
+	"strings"
+)
+
+type AggregateFunctionValuer struct{
+	Data AggregateData
+}
+
+func (v AggregateFunctionValuer) Value(key string) (interface{}, bool) {
+	return nil, false
+}
+
+func (v AggregateFunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
+	lowerName := strings.ToLower(name)
+	switch lowerName {
+	case "avg":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0{
+			v := getFirstValidArg(arg0)
+			switch v.(type){
+			case int:
+				return sliceIntTotal(arg0)/len(arg0), true
+			case int64:
+				return sliceIntTotal(arg0)/len(arg0), true
+			case float64:
+				return sliceFloatTotal(arg0)/float64(len(arg0)), true
+			default:
+				return fmt.Errorf("invalid data type for avg function"), false
+			}
+		}
+		return 0, true
+	case "count":
+		arg0 := args[0].([]interface{})
+		return len(arg0), true
+	case "max":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0{
+			v := getFirstValidArg(arg0)
+			switch t := v.(type){
+			case int:
+				return sliceIntMax(arg0, t), true
+			case int64:
+				return sliceIntMax(arg0, int(t)), true
+			case float64:
+				return sliceFloatMax(arg0, t), true
+			case string:
+				return sliceStringMax(arg0, t), true
+			default:
+				return fmt.Errorf("unsupported data type for avg function"), false
+			}
+		}
+		return fmt.Errorf("empty data for max function"), false
+	case "min":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0{
+			v := getFirstValidArg(arg0)
+			switch t := v.(type){
+			case int:
+				return sliceIntMin(arg0, t), true
+			case int64:
+				return sliceIntMin(arg0, int(t)), true
+			case float64:
+				return sliceFloatMin(arg0, t), true
+			case string:
+				return sliceStringMin(arg0, t), true
+			default:
+				return fmt.Errorf("unsupported data type for avg function"), false
+			}
+		}
+		return fmt.Errorf("empty data for max function"), false
+	case "sum":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0{
+			v := getFirstValidArg(arg0)
+			switch v.(type){
+			case int:
+				return sliceIntTotal(arg0), true
+			case int64:
+				return sliceIntTotal(arg0), true
+			case float64:
+				return sliceFloatTotal(arg0), true
+			default:
+				return fmt.Errorf("invalid data type for sum function"), false
+			}
+		}
+		return 0, true
+	default:
+		return nil, false
+	}
+}
+
+func (v *AggregateFunctionValuer) GetAllTuples() AggregateData {
+	return v.Data
+}
+
+func getFirstValidArg(s []interface{}) interface{}{
+	for _, v := range s{
+		if v != nil{
+			return v
+		}
+	}
+	return nil
+}
+
+func sliceIntTotal(s []interface{}) int{
+	var total int
+	for _, v := range s{
+		if v, ok := v.(int); ok {
+			total += v
+		}
+	}
+	return total
+}
+
+func sliceFloatTotal(s []interface{}) float64{
+	var total float64
+	for _, v := range s{
+		if v, ok := v.(float64); ok {
+			total += v
+		}
+	}
+	return total
+}
+func sliceIntMax(s []interface{}, max int) int{
+	for _, v := range s{
+		if v, ok := v.(int); ok {
+			if max < v {
+				max = v
+			}
+		}
+	}
+	return max
+}
+func sliceFloatMax(s []interface{}, max float64) float64{
+	for _, v := range s{
+		if v, ok := v.(float64); ok {
+			if max < v {
+				max = v
+			}
+		}
+	}
+	return max
+}
+
+func sliceStringMax(s []interface{}, max string) string{
+	for _, v := range s{
+		if v, ok := v.(string); ok {
+			if max < v {
+				max = v
+			}
+		}
+	}
+	return max
+}
+func sliceIntMin(s []interface{}, min int) int{
+	for _, v := range s{
+		if v, ok := v.(int); ok {
+			if min > v {
+				min = v
+			}
+		}
+	}
+	return min
+}
+func sliceFloatMin(s []interface{}, min float64) float64{
+	for _, v := range s{
+		if v, ok := v.(float64); ok {
+			if min > v {
+				min = v
+			}
+		}
+	}
+	return min
+}
+
+func sliceStringMin(s []interface{}, min string) string{
+	for _, v := range s{
+		if v, ok := v.(string); ok {
+			if min < v {
+				min = v
+			}
+		}
+	}
+	return min
+}

xsql/funcs_ast_validator.go

@@ -0,0 +1,318 @@
+package xsql
+
+import (
+	"fmt"
+	"strings"
+)
+
+type AllowTypes struct {
+	types []Literal
+}
+
+func validateFuncs(funcName string, args []Expr) (error) {
+	lowerName := strings.ToLower(funcName)
+	if _, ok := mathFuncMap[lowerName]; ok {
+		return validateMathFunc(funcName, args)
+	} else if _, ok := strFuncMap[lowerName]; ok {
+		return validateStrFunc(funcName, args)
+	} else if _, ok := convFuncMap[lowerName]; ok {
+		return validateConvFunc(lowerName, args)
+	} else if _, ok := hashFuncMap[lowerName]; ok {
+		return validateHashFunc(lowerName, args)
+	} else if _, ok := otherFuncMap[lowerName]; ok {
+		return validateOtherFunc(lowerName, args)
+	}
+	return nil
+}
+
+func validateMathFunc(name string, args []Expr) (error) {
+	len := len(args)
+	switch name {
+	case "abs", "acos", "asin", "atan", "ceil", "cos", "cosh", "exp", "ln", "log", "round", "sign", "sin", "sinh",
+	"sqrt", "tan", "tanh" :
+		if err := validateLen(name, 1, len); err != nil {
+			return  err
+		}
+		if isStringArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "number - float or int")
+		}
+	case "bitand", "bitor", "bitxor":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+		if isFloatArg(args[0]) || isStringArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]){
+			return produceErrInfo(name, 0, "int")
+		}
+		if isFloatArg(args[1]) || isStringArg(args[1]) || isTimeArg(args[1]) || isBooleanArg(args[1]) {
+			return produceErrInfo(name, 1, "int")
+		}
+
+	case "bitnot":
+		if err := validateLen(name, 1, len); err != nil {
+			return  err
+		}
+		if isFloatArg(args[0]) || isStringArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0])  {
+			return produceErrInfo(name, 0, "int")
+		}
+
+	case "atan2", "mod", "power":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+		if isStringArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "number - float or int")
+		}
+		if isStringArg(args[1]) || isTimeArg(args[1]) || isBooleanArg(args[1]){
+			return produceErrInfo(name, 1, "number - float or int")
+		}
+
+	case "rand":
+		if err := validateLen(name, 0, len); err != nil {
+			return  err
+		}
+	}
+	return nil
+}
+
+func validateStrFunc(name string, args []Expr) (error) {
+	len := len(args)
+	switch name {
+	case "concat":
+		if len == 0 {
+			return fmt.Errorf("The arguments for %s should be at least one.\n", name)
+		}
+		for i, a := range args {
+			if isNumericArg(a) || isTimeArg(a) || isBooleanArg(a) {
+				return produceErrInfo(name, i, "string")
+			}
+		}
+	case "endswith", "indexof", "regexp_matches", "startswith":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+		for i := 0; i < 2; i++ {
+			if isNumericArg(args[i]) || isTimeArg(args[i])|| isBooleanArg(args[i]) {
+				return produceErrInfo(name, i, "string")
+			}
+		}
+	case "format_time":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+
+		if isNumericArg(args[0]) || isStringArg(args[0])|| isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "datetime")
+		}
+		if isNumericArg(args[1]) || isTimeArg(args[1])|| isBooleanArg(args[1]) {
+			return produceErrInfo(name, 1, "string")
+		}
+
+	case "regexp_replace":
+		if err := validateLen(name, 3, len); err != nil {
+			return  err
+		}
+		for i := 0; i < 3; i++ {
+			if isNumericArg(args[i]) || isTimeArg(args[i])|| isBooleanArg(args[i]) {
+				return produceErrInfo(name, i, "string")
+			}
+		}
+	case "length", "lower", "ltrim", "numbytes", "rtrim", "trim", "upper":
+		if err := validateLen(name, 1, len); err != nil {
+			return  err
+		}
+		if isNumericArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "string")
+		}
+	case "lpad", "rpad":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+		if isNumericArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "string")
+		}
+		if isFloatArg(args[1]) || isTimeArg(args[1]) || isBooleanArg(args[1]) || isStringArg(args[1]) {
+			return produceErrInfo(name, 1, "int")
+		}
+	case "substring":
+		if len != 2 && len != 3 {
+			return fmt.Errorf("the arguments for substring should be 2 or 3")
+		}
+		if isNumericArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "string")
+		}
+		for i := 1; i < len; i++ {
+			if isFloatArg(args[i]) || isTimeArg(args[i]) || isBooleanArg(args[i]) || isStringArg(args[i]) {
+				return produceErrInfo(name, i, "int")
+			}
+		}
+
+		if s, ok := args[1].(*IntegerLiteral); ok {
+			sv := s.Val
+			if sv < 0 {
+				return fmt.Errorf("The start index should not be a nagtive integer.")
+			}
+			if len == 3{
+				if e, ok1 := args[2].(*IntegerLiteral); ok1 {
+					ev := e.Val
+					if ev < sv {
+						return fmt.Errorf("The end index should be larger than start index.")
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func validateConvFunc(name string, args []Expr) (error) {
+	len := len(args)
+	switch name {
+	case "cast":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+		a := args[1]
+		if !isStringArg(a) {
+			return produceErrInfo(name, 1, "string")
+		}
+		if av, ok := a.(*StringLiteral); ok {
+			if !(av.Val == "bigint" || av.Val == "float" || av.Val == "string" || av.Val == "boolean" || av.Val == "datetime") {
+				return fmt.Errorf("Expect one of following value for the 2nd parameter: bigint, float, string, boolean, datetime.")
+			}
+		}
+	case "chr":
+		if err := validateLen(name, 1, len); err != nil {
+			return  err
+		}
+		if isFloatArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "int")
+		}
+	case "encode":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+
+		if isNumericArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "string")
+		}
+
+		a := args[1]
+		if !isStringArg(a) {
+			return produceErrInfo(name, 1, "string")
+		}
+		if av, ok := a.(*StringLiteral); ok {
+			if av.Val != "base64" {
+				return fmt.Errorf("Only base64 is supported for the 2nd parameter.")
+			}
+		}
+	case "trunc":
+		if err := validateLen(name, 2, len); err != nil {
+			return  err
+		}
+
+		if isTimeArg(args[0]) || isBooleanArg(args[0]) || isStringArg(args[0]) {
+			return produceErrInfo(name, 0, "number - float or int")
+		}
+
+		if isFloatArg(args[1]) || isTimeArg(args[1]) || isBooleanArg(args[1]) || isStringArg(args[1]) {
+			return produceErrInfo(name, 1, "int")
+		}
+	}
+	return nil
+}
+
+func validateHashFunc(name string, args []Expr) (error) {
+	len := len(args)
+	switch name {
+	case "md5", "sha1", "sha224", "sha256", "sha384", "sha512":
+		if err := validateLen(name, 1, len); err != nil {
+			return err
+		}
+
+		if isNumericArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) {
+			return produceErrInfo(name, 0, "string")
+		}
+	}
+	return nil
+}
+
+func validateOtherFunc(name string, args []Expr) (error) {
+	len := len(args)
+	switch name {
+	case "isNull":
+		if err := validateLen(name, 1, len); err != nil {
+			return err
+		}
+	case "nanvl":
+		if err := validateLen(name, 2, len); err != nil {
+			return err
+		}
+		if isIntegerArg(args[0]) || isTimeArg(args[0]) || isBooleanArg(args[0]) || isStringArg(args[0]) {
+			return produceErrInfo(name, 1, "float")
+		}
+	case "newuuid":
+		if err := validateLen(name, 0, len); err != nil {
+			return  err
+		}
+	}
+	return nil
+}
+
+
+// Index is starting from 0
+func produceErrInfo(name string, index int, expect string) (err error) {
+	index++
+	err = fmt.Errorf("Expect %s type for %d parameter of function %s.", expect, index, name)
+	return
+}
+
+func validateLen(funcName string, exp, actual int) (error) {
+	if actual != exp {
+		return fmt.Errorf("The arguments for %s should be %d.", funcName, exp)
+	}
+	return nil
+}
+
+func isNumericArg(arg Expr) bool {
+	if _, ok := arg.(*NumberLiteral); ok {
+		return true
+	} else if _, ok := arg.(*IntegerLiteral); ok {
+		return true
+	}
+	return false
+}
+
+func isIntegerArg(arg Expr) bool {
+	if _, ok := arg.(*IntegerLiteral); ok {
+		return true
+	}
+	return false
+}
+
+func isFloatArg(arg Expr) bool {
+	if _, ok := arg.(*NumberLiteral); ok {
+		return true
+	}
+	return false
+}
+
+func isBooleanArg(arg Expr) bool {
+	if _, ok := arg.(*BooleanLiteral); ok {
+		return true
+	}
+	return false
+}
+
+func isStringArg(arg Expr) bool {
+	if _, ok := arg.(*StringLiteral); ok {
+		return true
+	}
+	return false
+}
+
+func isTimeArg(arg Expr) bool {
+	if _, ok := arg.(*TimeLiteral); ok {
+		return true
+	}
+	return false
+}

xsql/funcs_ast_validator_test.go

@@ -0,0 +1,404 @@
+package xsql
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// Ensure the parser can parse strings into Statement ASTs.
+func TestFuncValidator(t *testing.T) {
+	var tests = []struct {
+		s    string
+		stmt *SelectStatement
+		err  string
+	}{
+		{
+			s: `SELECT abs(1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "abs", Expr:&Call{Name:"abs", Args: []Expr{&IntegerLiteral{Val:1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT abs(field1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "abs", Expr:&Call{Name:"abs", Args: []Expr{&FieldRef{Name:"field1"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT abs(1,2) FROM tbl`,
+			stmt: nil,
+			err: "The arguments for abs should be 1.",
+		},
+
+		{
+			s: `SELECT abs(1.1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "abs", Expr:&Call{Name:"abs", Args: []Expr{&NumberLiteral{Val:1.1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT abs(true) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function abs.",
+		},
+
+		{
+			s: `SELECT abs("test") FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function abs.",
+		},
+
+		{
+			s: `SELECT abs(ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function abs.",
+		},
+
+
+		///
+		{
+			s: `SELECT sin(1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "sin", Expr:&Call{Name:"sin", Args: []Expr{&IntegerLiteral{Val:1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT sin(1.1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "sin", Expr:&Call{Name:"sin", Args: []Expr{&NumberLiteral{Val:1.1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT sin(true) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function sin.",
+		},
+
+		{
+			s: `SELECT sin("test") FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function sin.",
+		},
+
+		{
+			s: `SELECT sin(ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function sin.",
+		},
+		///
+		{
+			s: `SELECT tanh(1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "tanh", Expr:&Call{Name:"tanh", Args: []Expr{&IntegerLiteral{Val:1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT tanh(1.1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "tanh", Expr:&Call{Name:"tanh", Args: []Expr{&NumberLiteral{Val:1.1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT tanh(true) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function tanh.",
+		},
+
+		{
+			s: `SELECT tanh("test") FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function tanh.",
+		},
+
+		{
+			s: `SELECT tanh(ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function tanh.",
+		},
+
+		///
+		{
+			s: `SELECT bitxor(1, 2) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "bitxor", Expr:&Call{Name:"bitxor", Args: []Expr{&IntegerLiteral{Val:1}, &IntegerLiteral{Val:2}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT bitxor(1.1, 2) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 1 parameter of function bitxor.",
+		},
+
+		{
+			s: `SELECT bitxor(true, 2) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 1 parameter of function bitxor.",
+		},
+
+		{
+			s: `SELECT bitxor(1, ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 2 parameter of function bitxor.",
+		},
+
+		{
+			s: `SELECT bitxor(1, 2.2) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 2 parameter of function bitxor.",
+		},
+
+		///
+		{
+			s: `SELECT bitnot(1) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "bitnot", Expr:&Call{Name:"bitnot", Args: []Expr{&IntegerLiteral{Val:1}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT bitnot(1.1) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 1 parameter of function bitnot.",
+		},
+
+		{
+			s: `SELECT bitnot(true) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 1 parameter of function bitnot.",
+		},
+
+		///
+		{
+			s: `SELECT mod(1, 2) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "mod", Expr:&Call{Name:"mod", Args: []Expr{&IntegerLiteral{Val:1}, &IntegerLiteral{Val:2}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT mod("1.1", 2) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 1 parameter of function mod.",
+		},
+
+		{
+			s: `SELECT mod(1.1, true) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 2 parameter of function mod.",
+		},
+
+		{
+			s: `SELECT mod(1, ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect number - float or int type for 2 parameter of function mod.",
+		},
+
+		///
+		{
+			s: `SELECT concat(field, "hello") FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "concat", Expr:&Call{Name:"concat", Args: []Expr{&FieldRef{Name:"field"}, &StringLiteral{Val:"hello"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT concat("1.1", 2) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 2 parameter of function concat.",
+		},
+
+		{
+			s: `SELECT concat("1.1", true) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 2 parameter of function concat.",
+		},
+
+		{
+			s: `SELECT concat("1", ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 2 parameter of function concat.",
+		},
+
+		///
+		{
+			s: `SELECT regexp_matches(field, "hello") FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "regexp_matches", Expr:&Call{Name:"regexp_matches", Args: []Expr{&FieldRef{Name:"field"}, &StringLiteral{Val:"hello"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT regexp_matches(1, "true") FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 1 parameter of function regexp_matches.",
+		},
+
+		{
+			s: `SELECT regexp_matches("1.1", 2) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 2 parameter of function regexp_matches.",
+		},
+
+		///
+		{
+			s: `SELECT regexp_replace(field, "hello", "h") FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "regexp_replace", Expr:&Call{Name:"regexp_replace", Args: []Expr{&FieldRef{Name:"field"}, &StringLiteral{Val:"hello"}, &StringLiteral{Val:"h"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT regexp_replace(field1, "true", true) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 3 parameter of function regexp_replace.",
+		},
+
+		///
+		{
+			s: `SELECT trim(field) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "trim", Expr:&Call{Name:"trim", Args: []Expr{&FieldRef{Name:"field"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT trim(1) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 1 parameter of function trim.",
+		},
+
+		///
+		{
+			s: `SELECT rpad(field, 3) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "rpad", Expr:&Call{Name:"rpad", Args: []Expr{&FieldRef{Name:"field"}, &IntegerLiteral{Val:3}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT rpad("ff", true) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 2 parameter of function rpad.",
+		},
+
+		///
+		{
+			s: `SELECT substring(field, 3, 4) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "substring", Expr:&Call{Name:"substring", Args: []Expr{&FieldRef{Name:"field"}, &IntegerLiteral{Val:3}, &IntegerLiteral{Val:4}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT substring(field, -1, 4) FROM tbl`,
+			stmt: nil,
+			err: "The start index should not be a nagtive integer.",
+		},
+
+		{
+			s: `SELECT substring(field, 0, -1) FROM tbl`,
+			stmt: nil,
+			err: "The end index should be larger than start index.",
+		},
+
+		{
+			s: `SELECT substring(field, 0, true) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 3 parameter of function substring.",
+		},
+
+		///
+		{
+			s: `SELECT cast(field, "bigint") FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "cast", Expr:&Call{Name:"cast", Args: []Expr{&FieldRef{Name:"field"}, &StringLiteral{Val:"bigint"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT cast("12", "bool") FROM tbl`,
+			stmt: nil,
+			err: "Expect one of following value for the 2nd parameter: bigint, float, string, boolean, datetime.",
+		},
+
+		///
+		{
+			s: `SELECT chr(field) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "chr", Expr:&Call{Name:"chr", Args: []Expr{&FieldRef{Name:"field"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT chr(true) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 1 parameter of function chr.",
+		},
+
+		///
+		{
+			s: `SELECT encode(field, "base64") FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "encode", Expr:&Call{Name:"encode", Args: []Expr{&FieldRef{Name:"field"}, &StringLiteral{Val:"base64"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT encode(field, true) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 2 parameter of function encode.",
+		},
+
+		///
+		{
+			s: `SELECT trunc(field, 3) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "trunc", Expr:&Call{Name:"trunc", Args: []Expr{&FieldRef{Name:"field"}, &IntegerLiteral{Val:3}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT trunc(5, ss) FROM tbl`,
+			stmt: nil,
+			err: "Expect int type for 2 parameter of function trunc.",
+		},
+
+		///
+		{
+			s: `SELECT sha512(field) FROM tbl`,
+			stmt: &SelectStatement{Fields: []Field{Field{ AName:"",  Name: "sha512", Expr:&Call{Name:"sha512", Args: []Expr{&FieldRef{Name:"field"}}}}},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT sha512(20) FROM tbl`,
+			stmt: nil,
+			err: "Expect string type for 1 parameter of function sha512.",
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		//fmt.Printf("Parsing SQL %q.\n", tt.s)
+		stmt, err := NewParser(strings.NewReader(tt.s)).Parse()
+		if !reflect.DeepEqual(tt.err, errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
+		} else if tt.err == "" && !reflect.DeepEqual(tt.stmt, stmt) {
+			t.Errorf("%d. %q\n\nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.s, tt.stmt, stmt)
+		}
+	}
+}
+
+

xsql/funcs_math.go

@@ -0,0 +1,200 @@
+package xsql
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+)
+
+func mathCall(name string, args []interface{}) (interface{}, bool) {
+	switch name {
+	case "abs":
+		if v, ok := args[0].(int); ok {
+			t := float64(v)
+			var ret int = int(math.Abs(t))
+			return ret, true
+		} else if v, ok := args[0].(float64); ok {
+			return math.Abs(v), true
+		} else {
+			return fmt.Errorf("only float64 & int type are supported"), false
+		}
+	case "acos":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Acos(v), true
+		} else {
+			return e, false
+		}
+	case "asin":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Asin(v), true
+		} else {
+			return e, false
+		}
+	case "atan":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Atan(v), true
+		} else {
+			return e, false
+		}
+	case "atan2":
+		if v1, e := toF64(args[0]); e == nil {
+			if v2, e1 := toF64(args[1]); e1 == nil {
+				return math.Atan2(v1, v2), true
+			} else {
+				return e1, false
+			}
+		} else {
+			return e, false
+		}
+	case "bitand":
+		v1, ok1 := args[0].(int)
+		v2, ok2 := args[1].(int)
+		if ok1 && ok2 {
+			return v1 & v2, true
+		} else {
+			return fmt.Errorf("Expect int type for both operands."), false
+		}
+	case "bitor":
+		v1, ok1 := args[0].(int)
+		v2, ok2 := args[1].(int)
+		if ok1 && ok2 {
+			return v1 | v2, true
+		} else {
+			return fmt.Errorf("Expect int type for both operands."), false
+		}
+	case "bitxor":
+		v1, ok1 := args[0].(int)
+		v2, ok2 := args[1].(int)
+		if ok1 && ok2 {
+			return v1 ^ v2, true
+		} else {
+			return fmt.Errorf("Expect int type for both operands."), false
+		}
+	case "bitnot":
+		v1, ok1 := args[0].(int)
+		if ok1 {
+			return ^v1, true
+		} else {
+			return fmt.Errorf("Expect int type for operand."), false
+		}
+	case "ceil":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Ceil(v), true
+		} else {
+			return e, false
+		}
+	case "cos":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Cos(v), true
+		} else {
+			return e, false
+		}
+	case "cosh":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Cosh(v), true
+		} else {
+			return e, false
+		}
+	case "exp":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Exp(v), true
+		} else {
+			return e, false
+		}
+	case "ln":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Log2(v), true
+		} else {
+			return e, false
+		}
+	case "log":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Log10(v), true
+		} else {
+			return e, false
+		}
+	case "mod":
+		if v, e := toF64(args[0]); e == nil {
+			if v1, e1 := toF64(args[1]); e == nil {
+				return math.Mod(v, v1), true
+			} else {
+				return e1, false
+			}
+		} else {
+			return e, false
+		}
+	case "power":
+		if v1, e := toF64(args[0]); e == nil {
+			if v2, e2 := toF64(args[1]); e2 == nil {
+				return math.Pow(v1, v2), true
+			} else {
+				return e2, false
+			}
+		} else {
+			return e, false
+		}
+	case "rand":
+		return rand.Float64(), true
+	case "round":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Round(v), true
+		} else {
+			return e, false
+		}
+	case "sign":
+		if v, e := toF64(args[0]); e == nil {
+			if v > 0 {
+				return 1, true
+			} else if v < 0 {
+				return -1, true
+			} else {
+				return 0, true
+			}
+		} else {
+			return e, false
+		}
+	case "sin":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Sin(v), true
+		} else {
+			return e, false
+		}
+	case "sinh":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Sinh(v), true
+		} else {
+			return e, false
+		}
+	case "sqrt":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Sqrt(v), true
+		} else {
+			return e, false
+		}
+
+	case "tan":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Tan(v), true
+		} else {
+			return e, false
+		}
+
+	case "tanh":
+		if v, e := toF64(args[0]); e == nil {
+			return math.Tanh(v), true
+		} else {
+			return e, false
+		}
+	}
+
+	return fmt.Errorf("Unknown math function name."), false
+}
+
+func toF64(arg interface{}) (float64, error) {
+	if v, ok := arg.(float64); ok {
+		return v, nil
+	} else if v, ok := arg.(int); ok {
+		return float64(v), nil
+	}
+	return 0, fmt.Errorf("only float64 & int type are supported")
+}

xsql/funcs_misc.go

@@ -0,0 +1,207 @@
+package xsql
+
+import (
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	b64 "encoding/base64"
+	"engine/common"
+	"fmt"
+	"github.com/google/uuid"
+	"hash"
+	"io"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func convCall(name string, args []interface{}) (interface{}, bool) {
+	switch name {
+	case "cast":
+		if v, ok := args[1].(string); ok {
+			v = strings.ToLower(v)
+			switch v {
+			case "bigint":
+				if v1, ok1 := args[0].(int); ok1 {
+					return v1, true
+				} else if v1, ok1 := args[0].(float64); ok1 {
+					return int(v1), true
+				} else if v1, ok1 := args[0].(string); ok1 {
+					if temp, err := strconv.Atoi(v1); err == nil {
+						return temp, true
+					} else {
+						return err, false
+					}
+				} else if v1, ok1 := args[0].(bool); ok1 {
+					if v1 {
+						return 1, true
+					} else {
+						return 0, true
+					}
+				} else {
+					return fmt.Errorf("Not supported type conversion."), false
+				}
+			case "float":
+				if v1, ok1 := args[0].(int); ok1 {
+					return float64(v1), true
+				} else if v1, ok1 := args[0].(float64); ok1 {
+					return v1, true
+				} else if v1, ok1 := args[0].(string); ok1 {
+					if temp, err := strconv.ParseFloat(v1, 64); err == nil {
+						return temp, true
+					} else {
+						return err, false
+					}
+				} else if v1, ok1 := args[0].(bool); ok1 {
+					if v1 {
+						return 1.0, true
+					} else {
+						return 0.0, true
+					}
+				} else {
+					return fmt.Errorf("Not supported type conversion."), false
+				}
+			case "string":
+				if v1, ok1 := args[0].(int); ok1 {
+					return string(v1), true
+				} else if v1, ok1 := args[0].(float64); ok1 {
+					return fmt.Sprintf("%g", v1), true
+				} else if v1, ok1 := args[0].(string); ok1 {
+					return v1, true
+				} else if v1, ok1 := args[0].(bool); ok1 {
+					if v1 {
+						return "true", true
+					} else {
+						return "false", true
+					}
+				} else {
+					return fmt.Errorf("Not supported type conversion."), false
+				}
+			case "boolean":
+				if v1, ok1 := args[0].(int); ok1 {
+					if v1 == 0 {
+						return false, true
+					} else {
+						return true, true
+					}
+				} else if v1, ok1 := args[0].(float64); ok1 {
+					if v1 == 0.0 {
+						return false, true
+					} else {
+						return true, true
+					}
+				} else if v1, ok1 := args[0].(string); ok1 {
+					if temp, err := strconv.ParseBool(v1); err == nil {
+						return temp, true
+					} else {
+						return err, false
+					}
+				} else if v1, ok1 := args[0].(bool); ok1 {
+					return v1, true
+				} else {
+					return fmt.Errorf("Not supported type conversion."), false
+				}
+			case "datetime":
+				return fmt.Errorf("Not supported type conversion."), false
+			default:
+				return fmt.Errorf("Unknow type, only support bigint, float, string, boolean and datetime."), false
+			}
+		} else {
+			return fmt.Errorf("Expect string type for the 2nd parameter."), false
+		}
+	case "chr":
+		if v, ok := args[0].(int); ok {
+			return rune(v), true
+		} else if v, ok := args[0].(float64); ok {
+			temp := int(v)
+			return rune(temp), true
+		} else if v, ok := args[0].(string); ok {
+			if len(v) > 1 {
+				return fmt.Errorf("Parameter length cannot larger than 1."), false
+			}
+			r := []rune(v)
+			return r[0], true
+		} else {
+			return fmt.Errorf("Only bigint, float and string type can be convert to char type."), false
+		}
+	case "encode":
+		if v, ok := args[1].(string); ok {
+			v = strings.ToLower(v)
+			if v == "base64" {
+				if v1, ok1 := args[0].(string); ok1 {
+					return b64.StdEncoding.EncodeToString([]byte(v1)), true
+				} else {
+					return fmt.Errorf("Only string type can be encoded."), false
+				}
+			} else {
+				return fmt.Errorf("Only base64 encoding is supported."), false
+			}
+		}
+	case "trunc":
+		var v0 float64
+		if v1, ok := args[0].(int); ok {
+			v0 = float64(v1)
+		} else if v1, ok := args[0].(float64); ok {
+			v0 = v1
+		} else {
+			return fmt.Errorf("Only int and float type can be truncated."), false
+		}
+		if v2, ok := args[1].(int); ok {
+			return toFixed(v0, v2), true
+		} else {
+			return fmt.Errorf("The 2nd parameter must be int value."), false
+		}
+	default:
+		return fmt.Errorf("Not supported function name %s", name), false
+	}
+	return nil, false
+}
+
+func round(num float64) int {
+	return int(num + math.Copysign(0.5, num))
+}
+
+func toFixed(num float64, precision int) float64 {
+	output := math.Pow(10, float64(precision))
+	return float64(round(num * output)) / output
+}
+
+func hashCall(name string, args []interface{}) (interface{}, bool) {
+	arg0 := common.ToString(args[0])
+	var h hash.Hash
+	switch name {
+	case "md5":
+		h = md5.New()
+	case "sha1":
+		h = sha1.New()
+	case "sha256":
+		h = sha256.New()
+	case "sha384":
+		h = sha512.New384()
+	case "sha512":
+		h = sha512.New()
+	default:
+		return fmt.Errorf("unknown hash function name %s", name), false
+	}
+	io.WriteString(h, arg0)
+	return fmt.Sprintf("%x", h.Sum(nil)), true
+}
+
+func otherCall(name string, args []interface{}) (interface{}, bool) {
+	switch name {
+	case "isnull":
+		return args[0] == nil, true
+	case "newuuid":
+		if uuid, err := uuid.NewUUID(); err != nil {
+			return err, false
+		}else{
+			return uuid.String(), true
+		}
+	case "timestamp":
+		return common.TimeToUnixMilli(time.Now()), true
+	default:
+		return fmt.Errorf("unknown function name %s", name), false
+	}
+}

xsql/funcs_str.go

@@ -0,0 +1,115 @@
+package xsql
+
+import (
+	"bytes"
+	"engine/common"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+func strCall(name string, args []interface{}) (interface{}, bool) {
+	switch name {
+	case "concat":
+		var b bytes.Buffer
+		for _, arg := range args {
+			b.WriteString(common.ToString(arg))
+		}
+		return b.String(), true
+	case "endswith":
+		arg0, arg1 := common.ToString(args[0]), common.ToString(args[1])
+		return strings.HasSuffix(arg0, arg1), true
+	case "indexof":
+		arg0, arg1 := common.ToString(args[0]), common.ToString(args[1])
+		return strings.Index(arg0, arg1), true
+	case "length":
+		arg0 := common.ToString(args[0])
+		return utf8.RuneCountInString(arg0), true
+	case "lower":
+		arg0 := common.ToString(args[0])
+		return strings.ToLower(arg0), true
+	case "lpad":
+		arg0 := common.ToString(args[0])
+		arg1, err := common.ToInt(args[1])
+		if err != nil{
+			return err, false
+		}
+		return strings.Repeat(" ", arg1) + arg0, true
+	case "ltrim":
+		arg0 := common.ToString(args[0])
+		return strings.TrimLeftFunc(arg0, unicode.IsSpace), true
+	case "numbytes":
+		arg0 := common.ToString(args[0])
+		return len(arg0), true
+	case "format_time":
+		arg0 := args[0]
+		if t, ok := arg0.(time.Time); ok{
+			arg1 := common.ToString(args[1])
+			if s, err := common.FormatTime(t, arg1); err==nil{
+				return s, true
+			}
+		}
+		return "", false
+	case "regexp_matches":
+		arg0, arg1 := common.ToString(args[0]), common.ToString(args[1])
+		if matched, err := regexp.MatchString(arg1, arg0); err != nil{
+			return err, false
+		}else{
+			return matched, true
+		}
+	case "regexp_replace":
+		arg0, arg1, arg2 := common.ToString(args[0]), common.ToString(args[1]), common.ToString(args[2])
+		if re, err := regexp.Compile(arg1); err != nil{
+			return err, false
+		}else{
+			return re.ReplaceAllString(arg0, arg2), true
+		}
+	case "regexp_substr":
+		arg0, arg1 := common.ToString(args[0]), common.ToString(args[1])
+		if re, err := regexp.Compile(arg1); err != nil{
+			return err, false
+		}else{
+			return re.FindString(arg0), true
+		}
+	case "rpad":
+		arg0 := common.ToString(args[0])
+		arg1, err := common.ToInt(args[1])
+		if err != nil{
+			return err, false
+		}
+		return arg0 + strings.Repeat(" ", arg1), true
+	case "rtrim":
+		arg0 := common.ToString(args[0])
+		return strings.TrimRightFunc(arg0, unicode.IsSpace), true
+	case "substring":
+		arg0 := common.ToString(args[0])
+		arg1, err := common.ToInt(args[1])
+		if err != nil{
+			return err, false
+		}
+		if len(args) > 2{
+			arg2, err := common.ToInt(args[2])
+			if err != nil{
+				return err, false
+			}
+			return arg0[arg1:arg2], true
+		}else{
+			return arg0[arg1:], true
+		}
+	case "startswith":
+		arg0, arg1 := common.ToString(args[0]), common.ToString(args[1])
+		return strings.HasPrefix(arg0, arg1), true
+	case "trim":
+		arg0 := common.ToString(args[0])
+		return strings.TrimSpace(arg0), true
+	case "upper":
+		arg0 := common.ToString(args[0])
+		return strings.ToUpper(arg0), true
+	default:
+		return fmt.Errorf("unknown string function name %s", name), false
+	}
+}
+

xsql/functions.go

@@ -1,31 +1,70 @@
 package xsql
 
 import (
-	"math"
 	"strings"
 )
 
 type FunctionValuer struct{}
 
-var _ CallValuer = FunctionValuer{}
-
-func (FunctionValuer) Value(key string) (interface{}, bool) {
+func (*FunctionValuer) Value(key string) (interface{}, bool) {
 	return nil, false
 }
 
-func (FunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
+var aggFuncMap = map[string]string{"avg": "",
+	"count": "",
+	"max": "", "min": "",
+	"sum":  "",
+}
+
+var mathFuncMap = map[string]string{"abs": "", "acos": "", "asin": "", "atan": "", "atan2": "",
+	"bitand": "", "bitor": "", "bitxor": "", "bitnot": "",
+	"ceil": "", "cos": "", "cosh": "",
+	"exp": "",
+	"ln":  "", "log": "",
+	"mod":   "",
+	"power": "",
+	"rand":  "", "round": "",
+	"sign": "", "sin": "", "sinh": "", "sqrt": "",
+	"tan": "", "tanh": "",
+}
+
+var strFuncMap = map[string]string{"concat": "",
+	"endswith": "",
+	"format_time": "",
+	"indexof":  "",
+	"length":   "", "lower": "", "lpad": "", "ltrim": "",
+	"numbytes":       "",
+	"regexp_matches": "", "regexp_replace": "", "regexp_substr": "", "rpad": "", "rtrim": "",
+	"substring": "", "startswith": "",
+	"trim":  "",
+	"upper": "",
+}
+
+var convFuncMap = map[string]string{"concat": "", "cast": "", "chr": "",
+	"encode": "",
+	"trunc":  "",
+}
+
+var hashFuncMap = map[string]string{ "md5": "",
+	"sha1": "", "sha256": "", "sha384": "", "sha512": "",
+}
+
+var otherFuncMap = map[string]string{"isNull": "",
+	"newuuid": "", "timestamp": "",
+}
+
+func (*FunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
 	lowerName := strings.ToLower(name)
-	switch lowerName {
-	case "round":
-		arg0 := args[0].(float64)
-		return math.Round(arg0), true
-	case "abs":
-		arg0 := args[0].(float64)
-		return math.Abs(arg0), true
-	case "pow":
-		arg0, arg1 := args[0].(float64), args[1].(int64)
-		return math.Pow(arg0, float64(arg1)), true
-	default:
-		return nil, false
+	if _, ok := mathFuncMap[lowerName]; ok {
+		return mathCall(name, args)
+	} else if _, ok := strFuncMap[lowerName]; ok {
+		return strCall(lowerName, args)
+	} else if _, ok := convFuncMap[lowerName]; ok {
+		return convCall(lowerName, args)
+	} else if _, ok := hashFuncMap[lowerName]; ok {
+		return hashCall(lowerName, args)
+	} else if _, ok := otherFuncMap[lowerName]; ok {
+		return otherCall(lowerName, args)
 	}
-}
+	return nil, false
+}

xsql/lexical.go

@@ -67,12 +67,16 @@ const (
 	SELECT
 	FROM
 	JOIN
-	LEFT
 	INNER
+	LEFT
+	RIGHT
+	FULL
+	CROSS
 	ON
 	WHERE
 	GROUP
 	ORDER
+	HAVING
 	BY
 	ASC
 	DESC
@@ -103,6 +107,8 @@ const (
 	CONF_KEY
 	TYPE
 	STRICT_VALIDATION
+	TIMESTAMP
+	TIMESTAMP_FORMAT
 
 	DD
 	HH
@@ -160,6 +166,7 @@ var tokens = []string{
 	WHERE:  "WHERE",
 	GROUP:  "GROUP",
 	ORDER:  "ORDER",
+	HAVING:  "HAVING",
 	BY:     "BY",
 	ASC:    "ASC",
 	DESC:   "DESC",
@@ -187,6 +194,8 @@ var tokens = []string{
 	CONF_KEY: "CONF_KEY",
 	TYPE: 	  "TYPE",
 	STRICT_VALIDATION: "STRICT_VALIDATION",
+	TIMESTAMP: "TIMESTAMP",
+	TIMESTAMP_FORMAT: "TIMESTAMP_FORMAT",
 
 	AND: "AND",
 	OR:  "OR",
@@ -362,6 +371,8 @@ func (s *Scanner) ScanIdent() (tok Token, lit string) {
 		return OR, lit
 	case "GROUP":
 		return GROUP, lit
+	case "HAVING":
+		return HAVING, lit
 	case "ORDER":
 		return ORDER, lit
 	case "BY":
@@ -374,6 +385,12 @@ func (s *Scanner) ScanIdent() (tok Token, lit string) {
 		return INNER, lit
 	case "LEFT":
 		return LEFT, lit
+	case "RIGHT":
+		return RIGHT, lit
+	case "FULL":
+		return FULL, lit
+	case "CROSS":
+		return CROSS, lit
 	case "JOIN":
 		return JOIN, lit
 	case "ON":
@@ -424,6 +441,10 @@ func (s *Scanner) ScanIdent() (tok Token, lit string) {
 		return FALSE, lit
 	case "STRICT_VALIDATION":
 		return STRICT_VALIDATION, lit
+	case "TIMESTAMP":
+		return TIMESTAMP, lit
+	case "TIMESTAMP_FORMAT":
+		return TIMESTAMP_FORMAT, lit
 	case "DD":
 		return DD, lit
 	case "HH":

xsql/parser.go

@@ -140,6 +140,12 @@ func (p *Parser) Parse() (*SelectStatement, error) {
 		selects.Dimensions = dims
 	}
 
+	if having, err := p.parseHaving(); err != nil {
+		return nil, err
+	} else {
+		selects.Having = having
+	}
+
 	if sorts, err := p.parseSorts(); err != nil {
 		return nil, err
 	} else {
@@ -202,7 +208,7 @@ func (p *Parser) parseSourceLiteral() (string, string, error) {
 func (p *Parser) parseFieldNameSections() ([]string, error) {
 	var fieldNameSects []string
 	for {
-		if tok, lit := p.scanIgnoreWhitespace(); tok == IDENT {
+		if tok, lit := p.scanIgnoreWhitespace(); tok == IDENT || tok == ASTERISK {
 			fieldNameSects = append(fieldNameSects, lit)
 			if tok1, _ := p.scanIgnoreWhitespace(); !tok1.allowedSFNToken() {
 				p.unscan()
@@ -224,16 +230,25 @@ func (p *Parser) parseFieldNameSections() ([]string, error) {
 func (p *Parser) parseJoins() (Joins, error) {
 	var joins Joins
 	for {
-		if tok, lit := p.scanIgnoreWhitespace(); tok == INNER || tok == LEFT {
+		if tok, lit := p.scanIgnoreWhitespace(); tok == INNER || tok == LEFT || tok == RIGHT || tok == FULL || tok == CROSS {
 			if tok1, _ := p.scanIgnoreWhitespace(); tok1 == JOIN {
-				if j, err := p.ParseJoin(); err != nil {
+				var jt JoinType = INNER_JOIN
+				switch tok {
+				case INNER:
+					jt = INNER_JOIN
+				case LEFT:
+					jt = LEFT_JOIN
+				case RIGHT:
+					jt = RIGHT_JOIN
+				case FULL:
+					jt = FULL_JOIN
+				case CROSS:
+					jt = CROSS_JOIN
+				}
+
+				if j, err := p.ParseJoin(jt); err != nil {
 					return nil, err
 				} else {
-					if tok == INNER {
-						j.JoinType = INNER_JOIN
-					} else if tok == LEFT {
-						j.JoinType = LEFT_JOIN
-					}
 					joins = append(joins, *j)
 				}
 			} else {
@@ -250,14 +265,17 @@ func (p *Parser) parseJoins() (Joins, error) {
 	return joins, nil
 }
 
-func (p *Parser) ParseJoin() (*Join, error) {
-	var j = &Join{}
+func (p *Parser) ParseJoin(joinType JoinType) (*Join, error) {
+	var j = &Join{ JoinType : joinType }
 	if src, alias, err := p.parseSourceLiteral(); err != nil {
 		return nil, err
 	} else {
 		j.Name = src
 		j.Alias = alias
 		if tok1, _ := p.scanIgnoreWhitespace(); tok1 == ON {
+			if CROSS_JOIN == joinType {
+				return nil, fmt.Errorf("On expression is not required for cross join type.\n")
+			}
 			if exp, err := p.ParseExpr(); err != nil {
 				return nil, err
 			} else {
@@ -297,6 +315,20 @@ func (p *Parser) parseDimensions() (Dimensions, error) {
 	return ds, nil
 }
 
+
+func (p *Parser) parseHaving() (Expr, error) {
+	if tok, _ := p.scanIgnoreWhitespace(); tok != HAVING {
+		p.unscan()
+		return nil, nil
+	}
+	expr, err := p.ParseExpr()
+	if err != nil {
+		return nil, err
+	}
+	return expr, nil
+}
+
+
 func (p *Parser) parseSorts() (SortFields, error) {
 	var ss SortFields
 	if t, _ := p.scanIgnoreWhitespace(); t == ORDER {
@@ -557,9 +589,17 @@ func (p *Parser) parseCall(name string) (Expr, error) {
 	for {
 		if tok, _ := p.scanIgnoreWhitespace(); tok == RPAREN {
 			return &Call{Name: name, Args: args}, nil
+		} else if tok == ASTERISK {
+			if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 != RPAREN {
+				return nil, fmt.Errorf("found %q, expected right paren.", lit2)
+			} else {
+				args = append(args, &StringLiteral{Val:"*"})
+				return &Call{Name: name, Args: args}, nil
+			}
 		} else {
 			p.unscan()
 		}
+
 		if exp, err := p.ParseExpr(); err != nil {
 			return nil, err
 		} else {
@@ -576,6 +616,9 @@ func (p *Parser) parseCall(name string) (Expr, error) {
 		return nil, fmt.Errorf("found function call %q, expected ), but with %q.", name, lit)
 	}
 	if wt, error := validateWindows(name, args); wt == NOT_WINDOW {
+		if valErr := validateFuncs(name, args); valErr != nil {
+			return nil, valErr
+		}
 		return &Call{Name: name, Args: args}, nil
 	} else {
 		if error != nil {
@@ -616,17 +659,43 @@ func validateWindow(funcName string, expectLen int, args []Expr) (error) {
 	if len(args) != expectLen {
 		return fmt.Errorf("The arguments for %s should be %d.\n", funcName, expectLen)
 	}
-	if _, ok := args[0].(*TimeLiteral); ok {
-		return nil
-	} else {
+	if _, ok := args[0].(*TimeLiteral); !ok {
 		return fmt.Errorf("The 1st argument for %s is expecting timer literal expression. One value of [dd|hh|mi|ss|ms].\n", funcName)
 	}
-}
 
-func (p *Parser) ConvertToWindows(wtype WindowType, name string, args []Expr) (*Windows, error) {
-	win := &Windows{WindowType:wtype}
+	for i := 1; i< len(args); i++ {
+		if _, ok := args[i].(*IntegerLiteral); !ok {
+			return fmt.Errorf("The %d argument for %s is expecting interger literal expression. \n", i, funcName)
+		}
+	}
+	return nil
+
+}
 
-	win.Args = args
+func (p *Parser) ConvertToWindows(wtype WindowType, name string, args []Expr) (*Window, error) {
+	win := &Window{WindowType: wtype}
+	var unit = 1
+	v := args[0].(*TimeLiteral).Val
+	switch v{
+	case DD:
+		unit = 24 * 3600 * 1000
+	case HH:
+		unit = 3600 * 1000
+	case MI:
+		unit = 60 * 1000
+	case SS:
+		unit = 1000
+	case MS:
+		unit = 1
+	default:
+		return nil, fmt.Errorf("Invalid timeliteral %s", v)
+	}
+	win.Length = &IntegerLiteral{Val :  args[1].(*IntegerLiteral).Val * unit}
+	if len(args) > 2{
+		win.Interval = &IntegerLiteral{Val : args[2].(*IntegerLiteral).Val * unit}
+	}else{
+		win.Interval = &IntegerLiteral{Val: 0}
+	}
 	return win, nil
 }
 
@@ -893,7 +962,7 @@ func (p *Parser) parseStreamOptions() (map[string]string, error) {
 	if tok, lit := p.scanIgnoreWhitespace(); tok == LPAREN {
 		lStack.Push(LPAREN)
 		for {
-			if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == DATASOURCE || tok1 == FORMAT || tok1 == KEY || tok1 == CONF_KEY || tok1 == STRICT_VALIDATION || tok1 == TYPE {
+			if tok1, lit1 := p.scanIgnoreWhitespace(); tok1 == DATASOURCE || tok1 == FORMAT || tok1 == KEY || tok1 == CONF_KEY || tok1 == STRICT_VALIDATION || tok1 == TYPE || tok1 == TIMESTAMP || tok1 == TIMESTAMP_FORMAT {
 				if tok2, lit2 := p.scanIgnoreWhitespace(); tok2 == EQ {
 					if tok3, lit3 := p.scanIgnoreWhitespace(); tok3 == STRING {
 						if tok1 == STRICT_VALIDATION {

xsql/parser_test.go

@@ -378,6 +378,29 @@ func TestParser_ParseStatement(t *testing.T) {
 			},
 		},
 
+		{
+			s: `SELECT count(*) FROM tbl`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{
+						AName:"",
+						Name: "count",
+						Expr:&Call{
+							Name:"count",
+							Args: []Expr{&StringLiteral{Val:"*"}},
+						},
+					},
+				},
+				Sources: []Source{&Table{Name:"tbl"}},
+			},
+		},
+
+		{
+			s: `SELECT count(*, f1) FROM tbl`,
+			stmt: nil,
+			err:`found ",", expected right paren.`,
+		},
+
 
 		{
 			s: `SELECT "abc" FROM tbl`,
@@ -821,6 +844,33 @@ func TestParser_ParseStatement(t *testing.T) {
 		},
 
 		{
+			s: `SELECT temp AS t, name FROM topic/sensor1 WHERE name = "dname" GROUP BY name HAVING count(name) > 3`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{ Expr:&FieldRef{Name: "temp"}, Name: "temp", AName:"t"},
+					Field{ Expr:&FieldRef{Name: "name"}, Name: "name", AName:""},
+				},
+				Sources: []Source{&Table{Name:"topic/sensor1"}},
+				Condition: &BinaryExpr{LHS: &FieldRef{Name: "name"}, OP: EQ, RHS: &StringLiteral{Val: "dname"},},
+				Dimensions:Dimensions{Dimension{Expr:&FieldRef{Name:"name"}}},
+				Having: &BinaryExpr{LHS:&Call{Name:"count", Args:[]Expr{&FieldRef{StreamName:"", Name:"name"}}}, OP: GT, RHS: &IntegerLiteral{Val:3}},
+			},
+		},
+
+		{
+			s: `SELECT temp AS t, name FROM topic/sensor1 WHERE name = "dname" HAVING count(name) > 3`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{ Expr:&FieldRef{Name: "temp"}, Name: "temp", AName:"t"},
+					Field{ Expr:&FieldRef{Name: "name"}, Name: "name", AName:""},
+				},
+				Sources: []Source{&Table{Name:"topic/sensor1"}},
+				Condition: &BinaryExpr{LHS: &FieldRef{Name: "name"}, OP: EQ, RHS: &StringLiteral{Val: "dname"},},
+				Having: &BinaryExpr{LHS:&Call{Name:"count", Args:[]Expr{&FieldRef{StreamName:"", Name:"name"}}}, OP: GT, RHS: &IntegerLiteral{Val:3}},
+			},
+		},
+
+		{
 			s: `SELECT s1.temp AS t, name FROM topic/sensor1 AS s1 WHERE t = "dname" GROUP BY s1.temp`,
 			stmt: &SelectStatement{
 				Fields:    []Field{
@@ -1176,9 +1226,10 @@ func TestParser_ParseWindowsExpr(t *testing.T) {
 				Sources: []Source{&Table{Name:"tbl"}},
 				Dimensions: Dimensions{
 					Dimension{
-						Expr:&Windows{
+						Expr:&Window{
 							WindowType: TUMBLING_WINDOW,
-							Args:[]Expr{&TimeLiteral{Val:SS}, &IntegerLiteral{Val:10}},
+							Length: &IntegerLiteral{Val:10000},
+							Interval: &IntegerLiteral{Val:0},
 						},
 					},
 				},
@@ -1197,9 +1248,10 @@ func TestParser_ParseWindowsExpr(t *testing.T) {
 				Sources: []Source{&Table{Name:"tbl"}},
 				Dimensions: Dimensions{
 					Dimension{
-						Expr:&Windows{
+						Expr:&Window{
 							WindowType: HOPPING_WINDOW,
-							Args:[]Expr{ &TimeLiteral{Val:MI}, &IntegerLiteral{Val:5}, &IntegerLiteral{Val:1}},
+							Length: &IntegerLiteral{Val:3e5},
+							Interval: &IntegerLiteral{Val:6e4},
 						},
 					},
 				},
@@ -1218,9 +1270,10 @@ func TestParser_ParseWindowsExpr(t *testing.T) {
 				Sources: []Source{&Table{Name:"tbl"}},
 				Dimensions: Dimensions{
 					Dimension{
-						Expr:&Windows{
+						Expr:&Window{
 							WindowType: SESSION_WINDOW,
-							Args:[]Expr{ &TimeLiteral{Val:HH}, &IntegerLiteral{Val:5}, &IntegerLiteral{Val:1}},
+							Length: &IntegerLiteral{Val:1.8e7},
+							Interval: &IntegerLiteral{Val:3.6e6},
 						},
 					},
 				},
@@ -1239,9 +1292,10 @@ func TestParser_ParseWindowsExpr(t *testing.T) {
 				Sources: []Source{&Table{Name:"tbl"}},
 				Dimensions: Dimensions{
 					Dimension{
-						Expr:&Windows{
+						Expr:&Window{
 							WindowType: SLIDING_WINDOW,
-							Args:[]Expr{ &TimeLiteral{Val:MS}, &IntegerLiteral{Val:5}},
+							Length: &IntegerLiteral{Val:5},
+							Interval: &IntegerLiteral{Val:0},
 						},
 					},
 				},
@@ -1456,6 +1510,19 @@ func TestParser_ParseJsonExpr(t *testing.T) {
 		},
 
 		{
+			s: `SELECT demo.* FROM demo`,
+			stmt: &SelectStatement{
+				Fields: []Field{
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("demo"), Name: "*"},
+						Name:  "*",
+						AName: ""},
+				},
+				Sources: []Source{&Table{Name: "demo"}},
+			},
+		},
+
+		{
 			s: `SELECT demo.children[2:]->first AS c FROM demo`,
 			stmt: &SelectStatement{
 				Fields: []Field{
@@ -1655,6 +1722,101 @@ func TestParser_ParseJoins(t *testing.T) {
 				},
 			},
 		},
+
+		{
+			s: `SELECT t1.name FROM topic/sensor1 AS t1 RIGHT JOIN topic1/sensor2 AS t2 ON t1.f=t2.k`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("t1"), Name:"name"},
+						Name: "name",
+						AName:""},
+				},
+				Sources: []Source{&Table{Name:"topic/sensor1", Alias:"t1"}},
+				Joins: []Join{
+					Join{
+						Name:"topic1/sensor2", Alias: "t2", JoinType: RIGHT_JOIN, Expr: &BinaryExpr{
+							LHS: &FieldRef{StreamName:StreamName("t1"), Name:"f"},
+							OP: EQ,
+							RHS:&FieldRef{StreamName:StreamName("t2"), Name:"k"},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			s: `SELECT t1.name FROM topic/sensor1 AS t1 FULL JOIN topic1/sensor2 AS t2 ON t1.f=t2.k`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("t1"), Name:"name"},
+						Name: "name",
+						AName:""},
+				},
+				Sources: []Source{&Table{Name:"topic/sensor1", Alias:"t1"}},
+				Joins: []Join{
+					Join{
+						Name:"topic1/sensor2", Alias: "t2", JoinType: FULL_JOIN, Expr: &BinaryExpr{
+							LHS: &FieldRef{StreamName:StreamName("t1"), Name:"f"},
+							OP: EQ,
+							RHS:&FieldRef{StreamName:StreamName("t2"), Name:"k"},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			s: `SELECT t1.name FROM topic/sensor1 AS t1 CROSS JOIN topic1/sensor2 AS t2 ON t1.f=t2.k`,
+			stmt: nil,
+			err: "On expression is not required for cross join type.\n",
+		},
+
+		{
+			s: `SELECT t1.name FROM topic/sensor1 AS t1 CROSS JOIN topic1/sensor2 AS t2`,
+			stmt: &SelectStatement{
+				Fields:    []Field{
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("t1"), Name:"name"},
+						Name: "name",
+						AName:""},
+				},
+				Sources: []Source{&Table{Name:"topic/sensor1", Alias:"t1"}},
+				Joins: []Join{
+					Join{
+						Name:"topic1/sensor2", Alias: "t2", JoinType: CROSS_JOIN, Expr: nil,
+					},
+				},
+			},
+		},
+
+		{
+			s: `SELECT demo.*, demo2.* FROM demo LEFT JOIN demo2 on demo.f1 = demo2.f2`,
+			stmt: &SelectStatement{
+				Fields: []Field{
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("demo"), Name: "*"},
+						Name:  "*",
+						AName: ""},
+					Field{
+						Expr: &FieldRef{StreamName:StreamName("demo2"), Name: "*"},
+						Name:  "*",
+						AName: ""},
+				},
+				Sources: []Source{&Table{Name: "demo"}},
+				Joins: []Join{
+					Join{
+						Name:"demo2", Alias: "", JoinType: LEFT_JOIN, Expr:  &BinaryExpr{
+							LHS: &FieldRef{StreamName:StreamName("demo"), Name:"f1"},
+							OP: EQ,
+							RHS:&FieldRef{StreamName:StreamName("demo2"), Name:"f2"},
+						},
+					},
+				},
+			},
+		},
+
 	}
 
 	fmt.Printf("The test bucket size is %d.\n\n", len(tests))

xsql/plans/aggregate_operator.go

@@ -0,0 +1,70 @@
+package plans
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+)
+
+type AggregatePlan struct {
+	Dimensions xsql.Dimensions
+}
+
+
+/**
+ *  input: *xsql.Tuple from preprocessor | xsql.WindowTuplesSet from windowOp | xsql.JoinTupleSets from joinOp
+ *  output: xsql.GroupedTuplesSet
+ */
+func (p *AggregatePlan) Apply(ctx context.Context, data interface{}) interface{} {
+	log := common.GetLogger(ctx)
+	log.Debugf("aggregate plan receive %s", data)
+	var ms []xsql.DataValuer
+	switch input := data.(type) {
+	case xsql.DataValuer:
+		ms = append(ms, input)
+	case xsql.WindowTuplesSet:
+		if len(input) != 1 {
+			log.Infof("WindowTuplesSet with multiple tuples cannot be evaluated")
+			return nil
+		}
+		ms = make([]xsql.DataValuer, len(input[0].Tuples))
+		for i, m := range input[0].Tuples {
+			//this is needed or it will always point to the last
+			t := m
+			ms[i] = &t
+		}
+	case xsql.JoinTupleSets:
+		ms = make([]xsql.DataValuer, len(input))
+		for i, m := range input {
+			t := m
+			ms[i] = &t
+		}
+	default:
+		log.Errorf("Expect xsql.Valuer or its array type.")
+		return nil
+	}
+
+	result := make(map[string]xsql.GroupedTuples)
+	for _, m := range ms {
+		var name string
+		ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(m, &xsql.FunctionValuer{})}
+		for _, d := range p.Dimensions {
+			name += fmt.Sprintf("%v,", ve.Eval(d.Expr))
+		}
+		if ts, ok := result[name]; !ok{
+			result[name] = xsql.GroupedTuples{m}
+		}else{
+			result[name] = append(ts, m)
+		}
+	}
+	if len(result) > 0{
+		g := make([]xsql.GroupedTuples, 0, len(result))
+		for _, v := range result {
+			g = append(g, v)
+		}
+		return xsql.GroupedTuplesSet(g)
+	}else{
+		return nil
+	}
+}

xsql/plans/aggregate_test.go

@@ -0,0 +1,289 @@
+package plans
+
+import (
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestAggregatePlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data interface{}
+		result xsql.GroupedTuplesSet
+	}{
+		{
+			sql: "SELECT abc FROM tbl group by abc",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(6),
+					"def" : "hello",
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "tbl",
+						Message: xsql.Message{
+							"abc" : int64(6),
+							"def" : "hello",
+						},
+					},
+				},
+			},
+		},
+
+
+		{
+			sql: "SELECT abc FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10), f1",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+					},
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+					},
+				},
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT abc FROM src1 GROUP BY id1, TUMBLINGWINDOW(ss, 10), f1",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+					},
+				},
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+					},
+				},
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 GROUP BY src2.f2, TUMBLINGWINDOW(ss, 10)",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+						},
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 GROUP BY TUMBLINGWINDOW(ss, 10), src1.f1",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+						},
+					},
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+						},
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 GROUP BY TUMBLINGWINDOW(ss, 10), src1.ts",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854515000),},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2","ts": common.TimeFromUnixMilli(1568854573431),},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1","ts": common.TimeFromUnixMilli(1568854515000),},},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854515000),},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+						},
+					},
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854515000),},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2","ts": common.TimeFromUnixMilli(1568854573431),},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		pp := &AggregatePlan{Dimensions:stmt.Dimensions.GetGroups()}
+		result := pp.Apply(nil, tt.data)
+		gr, ok := result.(xsql.GroupedTuplesSet)
+		if !ok {
+			t.Errorf("result is not GroupedTuplesSet")
+		}
+		if len(tt.result) != len(gr) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, gr)
+		}
+
+		for _, r := range tt.result{
+			matched := false
+			for _, gre := range gr{
+				if reflect.DeepEqual(r, gre){
+					matched = true
+				}
+			}
+			if !matched{
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, r)
+			}
+		}
+	}
+}

xsql/plans/filter_operator.go

@@ -10,24 +10,68 @@ type FilterPlan struct {
 	Condition xsql.Expr
 }
 
+/**
+  *  input: *xsql.Tuple from preprocessor | xsql.WindowTuplesSet from windowOp | xsql.JoinTupleSets from joinOp
+  *  output: *xsql.Tuple | xsql.WindowTuplesSet | xsql.JoinTupleSets
+ */
 func (p *FilterPlan) Apply(ctx context.Context, data interface{}) interface{} {
-	log := common.Log
-	var input map[string]interface{}
-	if d, ok := data.(map[string]interface{}); !ok {
-		log.Errorf("Expect map[string]interface{} type.\n")
-		return nil
-	} else {
-		input = d
-	}
-	log.Infof("filterplan receive %s", input)
-	ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(xsql.MapValuer(input), &xsql.FunctionValuer{})}
-	result, ok := ve.Eval(p.Condition).(bool)
-	if ok {
-		if result {
+	log := common.GetLogger(ctx)
+	log.Debugf("filter plan receive %s", data)
+	switch input := data.(type) {
+	case xsql.Valuer:
+		ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, &xsql.FunctionValuer{})}
+		result, ok := ve.Eval(p.Condition).(bool)
+		if ok {
+			if result {
+				return input
+			}
+		} else {
+			log.Errorf("invalid condition that returns non-bool value")
+		}
+	case xsql.WindowTuplesSet:
+		if len(input) != 1 {
+			log.Infof("WindowTuplesSet with multiple tuples cannot be evaluated")
+			return nil
+		}
+		ms := input[0].Tuples
+		r := ms[:0]
+		for _, v := range ms {
+			ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(&v, &xsql.FunctionValuer{})}
+			result, ok := ve.Eval(p.Condition).(bool)
+			if ok {
+				if result {
+					r = append(r, v)
+				}
+			} else {
+				log.Errorf("invalid condition that returns non-bool value")
+				return nil
+			}
+		}
+		if len(r) > 0 {
+			input[0].Tuples = r
 			return input
 		}
-	} else {
-		log.Errorf("invalid condition that returns non-bool value")
+	case xsql.JoinTupleSets:
+		ms := input
+		r := ms[:0]
+		for _, v := range ms {
+			ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(&v, &xsql.FunctionValuer{})}
+			result, ok := ve.Eval(p.Condition).(bool)
+			if ok {
+				if result {
+					r = append(r, v)
+				}
+			} else {
+				log.Errorf("invalid condition that returns non-bool value")
+				return nil
+			}
+		}
+		if len(r) > 0{
+			return r
+		}
+	default:
+		log.Errorf("Expect xsql.Valuer or its array type.")
+		return nil
 	}
 	return nil
 }

xsql/plans/filter_test.go

@@ -1,6 +1,7 @@
 package plans
 
 import (
+	"engine/common"
 	"engine/xsql"
 	"fmt"
 	"reflect"
@@ -11,39 +12,209 @@ import (
 func TestFilterPlan_Apply(t *testing.T) {
 	var tests = []struct {
 		sql  string
-		data map[string]interface{}
+		data interface{}
 		result interface{}
 	}{
 		{
 			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 AND abc < 20",
-			data: map[string]interface{}{
-				"a" : int64(6),
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"a" : int64(6),
+				},
 			},
 			result: nil,
 		},
 
 		{
+			sql: "SELECT * FROM tbl WHERE abc > def and abc <= ghi",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : common.TimeFromUnixMilli(1568854515000),
+					"def" : common.TimeFromUnixMilli(1568853515000),
+					"ghi" : common.TimeFromUnixMilli(1568854515000),
+				},
+			},
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : common.TimeFromUnixMilli(1568854515000),
+					"def" : common.TimeFromUnixMilli(1568853515000),
+					"ghi" : common.TimeFromUnixMilli(1568854515000),
+				},
+			},
+		},
+
+		{
 			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 AND abc < 20",
-			data: map[string]interface{}{
-				"abc" : int64(6),
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(6),
+				},
 			},
-			result: map[string]interface{}{
-				"abc" : int64(6),
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc": int64(6),
+				},
 			},
 		},
 
 		{
 			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 OR def = \"hello\"",
-			data: map[string]interface{}{
-				"abc" : int64(34),
-				"def" : "hello",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(34),
+					"def" : "hello",
+				},
+			},
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(34),
+					"def" : "hello",
+				},
+			},
+		},
+
+		{
+			sql: "SELECT abc FROM tbl WHERE abc > \"2019-09-19T00:55:15.000Z\"",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : common.TimeFromUnixMilli(1568854515678),
+					"def" : "hello",
+				},
+			},
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : common.TimeFromUnixMilli(1568854515678),
+					"def" : "hello",
+				},
+			},
+		},
+
+		{
+			sql: "SELECT abc FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{"id1" : 1, "f1" : "v1"},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{"id1" : 2, "f1" : "v2"},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{"id1" : 3, "f1" : "v1"},
+						},
+					},
+				},
 			},
-			result: map[string]interface{}{
-				"abc" : int64(34),
-				"def" : "hello",
+			result: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{"id1" : 1, "f1" : "v1"},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{"id1" : 3, "f1" : "v1"},
+						},
+					},
+				},
 			},
 		},
 
+		{
+			sql: "SELECT abc FROM src1 WHERE f1 = \"v8\" GROUP BY TUMBLINGWINDOW(ss, 10)",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1"},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2"},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1"},
+						},
+					},
+				},
+			},
+			result: nil,
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 WHERE src1.f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10)",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 WHERE src1.f1 = \"v22\" GROUP BY TUMBLINGWINDOW(ss, 10)",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: nil,
+		},
+
 	}
 
 	fmt.Printf("The test bucket size is %d.\n\n", len(tests))

xsql/plans/join_multi_test.go

@@ -0,0 +1,402 @@
+package plans
+
+import (
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestMultiJoinPlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql    string
+		data   xsql.WindowTuplesSet
+		result interface{}
+	}{
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 left join src3 on src2.id2 = src3.id3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 1, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 1, "f3" : "x1" },},
+					},
+				},
+
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v3" },},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 inner join src3 on src2.id2 = src3.id3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 1, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 1, "f3" : "x1" },},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 inner join src3 on src1.id1 = src3.id3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 5, "f1" : "v5" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 2, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 5, "f1" : "v5" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 full join src3 on src1.id1 = src3.id3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 5, "f1" : "v5" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 2, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+					},
+				},
+
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 5, "f1" : "v5" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+					},
+				},
+
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 2, "f3" : "x1" },},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 right join src3 on src2.id2 = src3.id3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 1, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 1, "f3" : "x1" },},
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+					},
+				},
+
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 cross join src3",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1" },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 5, "f1" : "v5" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src2",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 1, "f2" : "w1" },
+						},{
+							Emitter: "src2",
+							Message: xsql.Message{ "id2" : 4, "f2" : "w3" },
+						},
+					},
+				},
+
+				xsql.WindowTuples{
+					Emitter:"src3",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 2, "f3" : "x1" },
+						},{
+							Emitter: "src3",
+							Message: xsql.Message{ "id3" : 5, "f3" : "x5" },
+						},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 2, "f3" : "x1" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+					},
+				},
+
+				//xsql.JoinTuple{
+				//	Tuples: []xsql.Tuple{
+				//		{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1" },},
+				//		{Emitter: "src2", Message: xsql.Message{ "id2" : 1, "f2" : "w1" },},
+				//		{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+				//	},
+				//},
+
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 5, "f1" : "v5" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 2, "f3" : "x1" },},
+						{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+					},
+				},
+
+				//xsql.JoinTuple{
+				//	Tuples: []xsql.Tuple{
+				//		{Emitter: "src1", Message: xsql.Message{ "id1" : 5, "f1" : "v5" },},
+				//		{Emitter: "src3", Message: xsql.Message{ "id3" : 5, "f3" : "x5" },},
+				//	},
+				//},
+
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		if table, ok := stmt.Sources[0].(*xsql.Table); !ok{
+			t.Errorf("statement source is not a table")
+		}else{
+			pp := &JoinPlan{Joins: stmt.Joins, From: table}
+			result := pp.Apply(nil, tt.data)
+			if !reflect.DeepEqual(tt.result, result) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
+			}
+		}
+	}
+}

xsql/plans/join_operator.go

@@ -4,37 +4,280 @@ import (
 	"context"
 	"engine/common"
 	"engine/xsql"
+	"fmt"
 )
 
+//TODO join expr should only be the equal op between 2 streams like tb1.id = tb2.id
 type JoinPlan struct {
+	From *xsql.Table
 	Joins xsql.Joins
 }
 
+// input:  xsql.WindowTuplesSet from windowOp, window is required for join
+// output: xsql.JoinTupleSets
 func (jp *JoinPlan) Apply(ctx context.Context, data interface{}) interface{} {
-	var log = common.Log
-	var input xsql.MultiEmitterTuples
-	if d, ok := data.(xsql.MultiEmitterTuples ); !ok {
-		log.Errorf("Expect MultiEmitterTuples type.\n")
+	log := common.GetLogger(ctx)
+	var input xsql.WindowTuplesSet
+	if d, ok := data.(xsql.WindowTuplesSet); !ok {
+		log.Errorf("Expect WindowTuplesSet type.\n")
 		return nil
 	} else {
+		log.Debugf("join plan receive %v", d)
 		input = d
 	}
 
-	result := xsql.MergedEmitterTupleSets{}
+	result := xsql.JoinTupleSets{}
 
-	for _, join := range jp.Joins {
-		ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(input, &xsql.FunctionValuer{}), JoinType: join.JoinType}
-		v := ve.Eval(join.Expr)
-		if v1, ok := v.(xsql.MergedEmitterTupleSets); ok {
-			result = jp.mergeSet(v1, result)
+	for i, join := range jp.Joins {
+		if i == 0 {
+			v, err := jp.evalSet(input, join)
+			if err != nil {
+				fmt.Println(err)
+				return nil
+			}
+			result = v
+		} else {
+			r1, _ := jp.evalJoinSets(&result, input, join)
+			if v1, ok := r1.(xsql.JoinTupleSets); ok {
+				result = v1
+			}
 		}
 	}
+	if result.Len() <= 0 {
+		log.Debugf("join plan yields nothing")
+		return nil
+	}
 	return result
 }
 
+func getStreamNames(join *xsql.Join) ([]string, error) {
+	var srcs []string
+	xsql.WalkFunc(join, func(node xsql.Node) {
+		if f,ok := node.(*xsql.FieldRef); ok {
+			if string(f.StreamName) == "" {
+				return
+			}
+			srcs = append(srcs, string(f.StreamName))
+		}
+	});
+	if len(srcs) != 2 {
+		return nil, fmt.Errorf("Not correct join expression, it requires exactly 2 sources at ON expression.")
+	}
+	return srcs, nil
+}
+
+func (jp *JoinPlan) evalSet(input xsql.WindowTuplesSet, join xsql.Join) (xsql.JoinTupleSets, error) {
+	var leftStream, rightStream string
+
+	if join.JoinType != xsql.CROSS_JOIN {
+		streams, err := getStreamNames(&join)
+		if err != nil {
+			return nil, err
+		}
+		leftStream = streams[0]
+		rightStream = streams[1]
+	} else {
+		if jp.From.Alias == "" {
+			leftStream = jp.From.Name
+		} else {
+			leftStream = jp.From.Alias
+		}
+
+		if join.Alias == "" {
+			rightStream = join.Name
+		} else {
+			rightStream = join.Alias
+		}
+	}
+
+	var lefts, rights []xsql.Tuple
+
+	lefts = input.GetBySrc(leftStream)
+	rights = input.GetBySrc(rightStream)
+
+	sets := xsql.JoinTupleSets{}
+
+	if join.JoinType == xsql.RIGHT_JOIN {
+		return jp.evalSetWithRightJoin(input, join, false)
+	}
+	for _, left := range lefts {
+		merged := &xsql.JoinTuple{}
+		if join.JoinType == xsql.LEFT_JOIN || join.JoinType == xsql.FULL_JOIN || join.JoinType == xsql.CROSS_JOIN {
+			merged.AddTuple(left)
+		}
+		for _, right := range rights {
+			if join.JoinType == xsql.CROSS_JOIN {
+				merged.AddTuple(right)
+			} else {
+				temp := &xsql.JoinTuple{}
+				temp.AddTuple(left)
+				temp.AddTuple(right)
+				ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(temp, &xsql.FunctionValuer{})}
+				if r, ok := ve.Eval(join.Expr).(bool); ok {
+					if r {
+						if join.JoinType == xsql.INNER_JOIN {
+							merged.AddTuple(left)
+							merged.AddTuple(right)
+							sets = append(sets, *merged)
+							merged = &xsql.JoinTuple{}
+						}else{
+							merged.AddTuple(right)
+						}
+					}
+				} else {
+					common.Log.Infoln("Evaluation error for set.")
+				}
+			}
+		}
+		if len(merged.Tuples) > 0 {
+			sets = append(sets, *merged)
+		}
+	}
+
+	if join.JoinType == xsql.FULL_JOIN {
+		if rightJoinSet, err := jp.evalSetWithRightJoin(input, join, true); err == nil && len(rightJoinSet) > 0 {
+			for _, jt := range rightJoinSet {
+				sets = append(sets, jt)
+			}
+		}
+	}
+	return sets, nil
+}
+
+func (jp *JoinPlan) evalSetWithRightJoin(input xsql.WindowTuplesSet, join xsql.Join, excludeJoint bool) (xsql.JoinTupleSets, error) {
+	streams, err := getStreamNames(&join)
+	if err != nil {
+		return nil, err
+	}
+	leftStream := streams[0]
+	rightStream := streams[1]
+	var lefts, rights []xsql.Tuple
+
+	lefts = input.GetBySrc(leftStream)
+	rights = input.GetBySrc(rightStream)
+
+	sets := xsql.JoinTupleSets{}
+
+	for _, right := range rights {
+		merged := &xsql.JoinTuple{}
+		merged.AddTuple(right)
+		isJoint := false
+
+		for _, left := range lefts {
+			temp := &xsql.JoinTuple{}
+			temp.AddTuple(right)
+			temp.AddTuple(left)
+			ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(temp, &xsql.FunctionValuer{})}
+			if r, ok  := ve.Eval(join.Expr).(bool); ok {
+				if r {
+					merged.AddTuple(left)
+					isJoint = true
+				}
+			} else {
+				common.Log.Infoln("Evaluation error for set.")
+			}
+		}
+		if excludeJoint {
+			if len(merged.Tuples) > 0 && (!isJoint) {
+				sets = append(sets, *merged)
+			}
+		} else {
+			if len(merged.Tuples) > 0 {
+				sets = append(sets, *merged)
+			}
+		}
+	}
+	return sets, nil
+}
+
 
-func (jp *JoinPlan) mergeSet(set1 xsql.MergedEmitterTupleSets, set2 xsql.MergedEmitterTupleSets) xsql.MergedEmitterTupleSets {
-	return set1
+func (jp *JoinPlan) evalJoinSets(set *xsql.JoinTupleSets, input xsql.WindowTuplesSet, join xsql.Join) (interface{}, error)  {
+	var rightStream string
+	if join.Alias == "" {
+		rightStream = join.Name
+	} else {
+		rightStream = join.Alias
+	}
+
+	rights := input.GetBySrc(rightStream)
+
+	newSets := xsql.JoinTupleSets{}
+	if join.JoinType == xsql.RIGHT_JOIN {
+		return jp.evalRightJoinSets(set, input, join,false)
+	}
+	for _, left := range *set {
+		merged := &xsql.JoinTuple{}
+		if join.JoinType == xsql.LEFT_JOIN || join.JoinType == xsql.FULL_JOIN || join.JoinType == xsql.CROSS_JOIN {
+			merged.AddTuples(left.Tuples)
+		}
+		innerAppend := false
+		for _, right := range rights {
+			if join.JoinType == xsql.CROSS_JOIN {
+				merged.AddTuple(right)
+			} else {
+				ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(&left, &right, &xsql.FunctionValuer{})}
+				if r, ok := ve.Eval(join.Expr).(bool); ok {
+					if r {
+						if join.JoinType == xsql.INNER_JOIN && !innerAppend {
+							merged.AddTuples(left.Tuples)
+							innerAppend = true
+						}
+						merged.AddTuple(right)
+					}
+				}
+			}
+		}
+
+		if len(merged.Tuples) > 0 {
+			newSets = append(newSets, *merged)
+		}
+	}
+
+	if join.JoinType == xsql.FULL_JOIN {
+		if rightJoinSet, err := jp.evalRightJoinSets(set, input, join, true); err == nil && len(rightJoinSet) > 0 {
+			for _, jt := range rightJoinSet {
+				newSets = append(newSets, jt)
+			}
+		}
+	}
+
+	return newSets, nil
 }
 
+func (jp *JoinPlan) evalRightJoinSets(set *xsql.JoinTupleSets, input xsql.WindowTuplesSet, join xsql.Join, excludeJoint bool) (xsql.JoinTupleSets, error)  {
+	var rightStream string
+	if join.Alias == "" {
+		rightStream = join.Name
+	} else {
+		rightStream = join.Alias
+	}
+	rights := input.GetBySrc(rightStream)
 
+	newSets := xsql.JoinTupleSets{}
+
+	for _, right := range rights {
+		merged := &xsql.JoinTuple{}
+		merged.AddTuple(right)
+		isJoint := false
+		for _, left := range *set {
+			ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(&right, &left, &xsql.FunctionValuer{})}
+			if r, ok := ve.Eval(join.Expr).(bool); ok {
+				if r {
+					isJoint = true
+					merged.AddTuples(left.Tuples)
+				}
+			}
+		}
+
+		if excludeJoint {
+			if len(merged.Tuples) > 0  && (!isJoint) {
+				newSets = append(newSets, *merged)
+			}
+		} else {
+			if len(merged.Tuples) > 0 {
+				newSets = append(newSets, *merged)
+			}
+		}
+	}
+	return newSets, nil
+}

xsql/plans/math_func_test.go

@@ -0,0 +1,483 @@
+package plans
+
+import (
+	"encoding/json"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestMathAndConversionFunc_Apply1(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data *xsql.Tuple
+		result []map[string]interface{}
+	}{
+		{
+			sql: "SELECT abs(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : -1,
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1), //Actually it should be 1, it's caused by json Unmarshal method, which convert int to float64
+			}},
+		},
+
+		{
+			sql: "SELECT abs(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : -1.1,
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": 1.1,
+			}},
+		},
+
+		{
+			sql: "SELECT abs(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : 1.1,
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": 1.1,
+			}},
+		},
+
+		{
+			sql: "SELECT acos(1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT asin(1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1.5707963267948966),
+			}},
+		},
+
+		{
+			sql: "SELECT atan(1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0.7853981633974483),
+			}},
+		},
+
+		{
+			sql: "SELECT atan2(1,1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0.7853981633974483),
+			}},
+		},
+
+		{
+			sql: "SELECT bitand(1,1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT bitand(1.0,1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: nil,
+		},
+
+		{
+			sql: "SELECT bitor(1,1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT bitxor(1,1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT bitnot(1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(-2),
+			}},
+		},
+
+		{
+			sql: "SELECT ceil(1.6) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(2),
+			}},
+		},
+
+		{
+			sql: "SELECT cos(0) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT cosh(0) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT exp(1.2) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(3.3201169227365472),
+			}},
+		},
+
+		{
+			sql: "SELECT ln(1) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT log(10) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT mod(10, 3) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: "SELECT power(10, 3) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1000),
+			}},
+		},
+
+		{
+			sql: "SELECT round(10.2) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(10),
+			}},
+		},
+
+		{
+			sql: "SELECT sign(10.2) AS a, sign(-2) as b, sign(0) as c FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+				"b": float64(-1),
+				"c": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT sin(0) as a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT sinh(0) as a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT sqrt(4) as a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(2),
+			}},
+		},
+
+		{
+			sql: "SELECT tan(0) as a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: "SELECT tanh(1) as a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0.7615941559557649),
+			}},
+		},
+
+		{
+			sql: `SELECT cast(1.2, "bigint") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(1),
+			}},
+		},
+
+		{
+			sql: `SELECT cast(5, "bigint") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(5),
+			}},
+		},
+
+		{
+			sql: `SELECT cast(1.2, "string") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": "1.2",
+			}},
+		},
+
+		{
+			sql: `SELECT cast(true, "string") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": "true",
+			}},
+		},
+
+		{
+			sql: `SELECT cast("true", "boolean") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": true,
+			}},
+		},
+
+		{
+			sql: `SELECT cast("1", "boolean") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": true,
+			}},
+		},
+
+		{
+			sql: `SELECT cast(0.0, "boolean") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": false,
+			}},
+		},
+
+		{
+			sql: `SELECT chr(0) as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(0),
+			}},
+		},
+
+		{
+			sql: `SELECT chr("a") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(97),
+			}},
+		},
+
+		{
+			sql: `SELECT encode("hello", "base64") as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": "aGVsbG8=",
+			}},
+		},
+
+		{
+			sql: `SELECT trunc(3.1415, 2) as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(3.14),
+			}},
+		},
+
+		{
+			sql: `SELECT trunc(3, 2) as a FROM test`,
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: nil,
+			},
+			result: []map[string]interface{}{{
+				"a": float64(3.00),
+			}},
+		},
+
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		//fmt.Println("Running test " + strconv.Itoa(i))
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil && tt.result == nil {
+			continue
+		} else if err != nil && tt.result != nil{
+			t.Errorf("%q", err)
+			continue
+		}
+		pp := &ProjectPlan{Fields:stmt.Fields}
+		result := pp.Apply(nil, tt.data)
+		var mapRes []map[string]interface{}
+		if v, ok := result.([]byte); ok {
+			err := json.Unmarshal(v, &mapRes)
+			if err != nil {
+				t.Errorf("Failed to parse the input into map.\n")
+				continue
+			}
+			//fmt.Printf("%t\n", mapRes["rengine_field_0"])
+
+			if !reflect.DeepEqual(tt.result, mapRes) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, mapRes)
+			}
+		} else {
+			t.Errorf("The returned result is not type of []byte\n")
+		}
+	}
+}

xsql/plans/misc_func_test.go

@@ -0,0 +1,114 @@
+package plans
+
+import (
+	"encoding/json"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestHashFunc_Apply1(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data *xsql.Tuple
+		result []map[string]interface{}
+	}{
+		{
+			sql: "SELECT md5(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "The quick brown fox jumps over the lazy dog",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": strings.ToLower("9E107D9D372BB6826BD81D3542A419D6"),
+			}},
+		},
+		{
+			sql: "SELECT sha1(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "The quick brown fox jumps over the lazy dog",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": strings.ToLower("2FD4E1C67A2D28FCED849EE1BB76E7391B93EB12"),
+			}},
+		},
+		{
+			sql: "SELECT sha256(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "The quick brown fox jumps over the lazy dog",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": strings.ToLower("D7A8FBB307D7809469CA9ABCB0082E4F8D5651E46D3CDB762D02D0BF37C9E592"),
+			}},
+		},
+		{
+			sql: "SELECT sha384(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "The quick brown fox jumps over the lazy dog",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": strings.ToLower("CA737F1014A48F4C0B6DD43CB177B0AFD9E5169367544C494011E3317DBF9A509CB1E5DC1E85A941BBEE3D7F2AFBC9B1"),
+			}},
+		},
+		{
+			sql: "SELECT sha512(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "The quick brown fox jumps over the lazy dog",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": strings.ToLower("07E547D9586F6A73F73FBAC0435ED76951218FB7D0C8D788A309D785436BBB642E93A252A954F23912547D1E8A3B5ED6E1BFD7097821233FA0538F3DB854FEE6"),
+			}},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil || stmt == nil {
+			t.Errorf("parse sql %s error %v", tt.sql, err)
+		}
+		pp := &ProjectPlan{Fields:stmt.Fields}
+		result := pp.Apply(nil, tt.data)
+		var mapRes []map[string]interface{}
+		if v, ok := result.([]byte); ok {
+			err := json.Unmarshal(v, &mapRes)
+			if err != nil {
+				t.Errorf("Failed to parse the input into map.\n")
+				continue
+			}
+			//fmt.Printf("%t\n", mapRes["rengine_field_0"])
+
+			if !reflect.DeepEqual(tt.result, mapRes) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, mapRes)
+			}
+		} else {
+			t.Errorf("The returned result is not type of []byte\n")
+		}
+	}
+}

xsql/plans/order_operator.go

@@ -0,0 +1,32 @@
+package plans
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+)
+
+type OrderPlan struct {
+	SortFields xsql.SortFields
+}
+
+/**
+  *  input: *xsql.Tuple from preprocessor | xsql.WindowTuplesSet from windowOp | xsql.JoinTupleSets from joinOp
+  *  output: *xsql.Tuple | xsql.WindowTuplesSet | xsql.JoinTupleSets
+ */
+func (p *OrderPlan) Apply(ctx context.Context, data interface{}) interface{} {
+	log := common.GetLogger(ctx)
+	log.Debugf("order plan receive %s", data)
+	sorter := xsql.OrderedBy(p.SortFields)
+	switch input := data.(type) {
+	case xsql.Valuer:
+		return input
+	case xsql.SortingData:
+		sorter.Sort(input)
+		return input
+	default:
+		log.Errorf("Expect xsql.Valuer or its array type.")
+		return nil
+	}
+	return nil
+}

xsql/plans/order_test.go

@@ -0,0 +1,381 @@
+package plans
+
+import (
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestOrderPlan_Apply(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data interface{}
+		result interface{}
+	}{
+		{
+			sql: "SELECT * FROM tbl WHERE abc*2+3 > 12 AND abc < 20 ORDER BY abc",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(6),
+				},
+			},
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(6),
+				},
+			},
+		},
+
+		{
+			sql: "SELECT abc FROM tbl WHERE abc*2+3 > 12 OR def = \"hello\"",
+			data: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(34),
+					"def" : "hello",
+				},
+			},
+			result: &xsql.Tuple{
+				Emitter: "tbl",
+				Message: xsql.Message{
+					"abc" : int64(34),
+					"def" : "hello",
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY id1 DESC",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},
+					},
+				},
+			},
+			result: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT * FROM src1 WHERE f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY f1, id1 DESC",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},
+					},
+				},
+			},
+			result: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+						},
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT * FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY ts DESC",
+			data: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854515000)},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", "ts": common.TimeFromUnixMilli(1568854525000)},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854535000)},
+						},
+					},
+				},
+			},
+			result: xsql.WindowTuplesSet{
+				xsql.WindowTuples{
+					Emitter:"src1",
+					Tuples:[]xsql.Tuple{
+						{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 3, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854535000)},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 2, "f1" : "v2", "ts": common.TimeFromUnixMilli(1568854525000)},
+						},{
+							Emitter: "src1",
+							Message: xsql.Message{ "id1" : 1, "f1" : "v1", "ts": common.TimeFromUnixMilli(1568854515000)},
+						},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 WHERE src1.f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY src1.id1 desc",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT id1 FROM src1 left join src2 on src1.id1 = src2.id2 WHERE src1.f1 = \"v1\" GROUP BY TUMBLINGWINDOW(ss, 10) ORDER BY src2.id2",
+			data: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+			result: xsql.JoinTupleSets{
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+						{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+					},
+				},
+				xsql.JoinTuple{
+					Tuples: []xsql.Tuple{
+						{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+					},
+				},
+			},
+		},
+
+		{
+			sql: "SELECT abc FROM tbl group by abc ORDER BY def",
+			data: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "tbl",
+						Message: xsql.Message{
+							"abc" : int64(6),
+							"def" : "hello",
+						},
+					},
+				},
+			},
+			result:xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "tbl",
+						Message: xsql.Message{
+							"abc" : int64(6),
+							"def" : "hello",
+						},
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT id1 FROM src1 GROUP BY TUMBLINGWINDOW(ss, 10), f1 ORDER BY id1 desc",
+			data: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+					},
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+					},
+				},
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 2, "f1" : "v2", },
+					},
+				},
+				{
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 1, "f1" : "v1", },
+					},
+					&xsql.Tuple{
+						Emitter: "src1",
+						Message: xsql.Message{ "id1" : 3, "f1" : "v1", },
+					},
+				},
+			},
+		},
+		{
+			sql: "SELECT src2.id2 FROM src1 left join src2 on src1.id1 = src2.id2 GROUP BY src2.f2, TUMBLINGWINDOW(ss, 10) ORDER BY src2.id2 DESC",
+			data: xsql.GroupedTuplesSet{
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+						},
+					},
+				},
+			},
+			result: xsql.GroupedTuplesSet{
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 3, "f1" : "v1",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 2, "f1" : "v2",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 4, "f2" : "w3",},},
+						},
+					},
+				},
+				{
+					&xsql.JoinTuple{
+						Tuples: []xsql.Tuple{
+							{Emitter: "src1", Message: xsql.Message{ "id1" : 1, "f1" : "v1",},},
+							{Emitter: "src2", Message: xsql.Message{ "id2" : 2, "f2" : "w2",},},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil {
+			t.Errorf("statement parse error %s", err)
+			break
+		}
+
+		pp := &OrderPlan{SortFields:stmt.SortFields}
+		result := pp.Apply(nil, tt.data)
+		if !reflect.DeepEqual(tt.result, result) {
+			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)
+		}
+	}
+}

xsql/plans/preprocessor.go

@@ -2,141 +2,190 @@ package plans
 
 import (
 	"context"
-	"encoding/json"
 	"engine/common"
 	"engine/xsql"
-	"errors"
 	"fmt"
 	"reflect"
+	"strings"
+	"time"
 )
 
 type Preprocessor struct {
-	StreamStmt *xsql.StreamStmt
+	streamStmt  *xsql.StreamStmt
+	isEventTime bool
+	timestampField string
+	timestampFormat string
 }
 
-// data is a json string
+func NewPreprocessor(s *xsql.StreamStmt, iet bool) (*Preprocessor, error){
+	p := &Preprocessor{streamStmt: s, isEventTime: iet}
+	if iet {
+		if tf, ok := s.Options["TIMESTAMP"]; ok{
+			p.timestampField = tf
+		}else{
+			return nil, fmt.Errorf("preprocessor is set to be event time but stream option TIMESTAMP not found")
+		}
+		if ts, ok := s.Options["TIMESTAMP_FORMAT"]; ok{
+			p.timestampFormat = ts
+		}
+	}
+
+	return p, nil
+}
+
+/*
+ *	input: *xsql.Tuple
+ *	output: *xsql.Tuple
+ */
 func (p *Preprocessor) Apply(ctx context.Context, data interface{}) interface{} {
-	log := common.Log
+	log := common.GetLogger(ctx)
 	tuple, ok := data.(*xsql.Tuple)
 	if !ok {
-		log.Errorf("Expect tuple data type.\n")
+		log.Errorf("Expect tuple data type")
 		return nil
 	}
-	var jsonData []byte
-	if d, ok := tuple.Message.([]byte); !ok {
-		log.Errorf("Expect byte array data type.\n")
-		return nil
-	} else {
-		jsonData = d
-	}
-	log.Infof("preprocessor receive %s", jsonData)
-	//The unmarshal type can only be bool, float64, string, []interface{}, map[string]interface{}, nil
-	jsonResult := make(map[string]interface{})
+
+	log.Debugf("preprocessor receive %s", tuple.Message)
 
 	result := make(map[string]interface{})
-	if e := json.Unmarshal(jsonData, &jsonResult); e != nil {
-		log.Errorf("parse json string %s error: %s", jsonData, e)
-		return nil
-	}else{
-		for _, f := range p.StreamStmt.StreamFields {
-			if e = addRecField(f.FieldType, result, jsonResult, f.Name); e != nil{
-				log.Errorf("error in preprocessor: %s", e)
+	for _, f := range p.streamStmt.StreamFields {
+		fname := strings.ToLower(f.Name)
+		if e := p.addRecField(f.FieldType, result, tuple.Message, fname); e != nil{
+			log.Errorf("error in preprocessor: %s", e)
+			return nil
+		}
+	}
+	tuple.Message = result
+	if p.isEventTime{
+		if t, ok := result[p.timestampField]; ok{
+			if ts, err := common.InterfaceToUnixMilli(t, p.timestampFormat); err != nil{
+				log.Errorf("cannot convert timestamp field %s to timestamp with error %v", p.timestampField, err)
 				return nil
+			}else{
+				tuple.Timestamp = ts
+				log.Debugf("preprocessor calculate timstamp %d", tuple.Timestamp)
 			}
+		}else{
+			log.Errorf("cannot find timestamp field %s in tuple %v", p.timestampField, result)
+			return nil
 		}
-		tuple.Message = result
-		return tuple
 	}
+	return tuple
 }
 
-func addRecField(ft xsql.FieldType, r map[string]interface{}, j map[string]interface{}, p string) error {
-	if t, ok := j[p]; ok {
+func (p *Preprocessor) parseTime(s string) (time.Time, error){
+	if f, ok := p.streamStmt.Options["TIMESTAMP_FORMAT"]; ok{
+		return common.ParseTime(s, f)
+	}else{
+		return time.Parse(common.JSISO, s)
+	}
+}
+
+func (p *Preprocessor) addRecField(ft xsql.FieldType, r map[string]interface{}, j map[string]interface{}, n string) error {
+	if t, ok := j[n]; ok {
 		v := reflect.ValueOf(t)
 		jtype := v.Kind()
 		switch st := ft.(type) {
 		case *xsql.BasicType:
 			switch st.Type {
 			case xsql.UNKNOWN:
-				return errors.New(fmt.Sprintf("invalid data type unknown defined for %s, please check the stream definition", t))
+				return fmt.Errorf("invalid data type unknown defined for %s, please check the stream definition", t)
 			case xsql.BIGINT:
-				if jtype == reflect.Float64{
-					r[p] = int(t.(float64))
+				if jtype == reflect.Int{
+					r[n] = t.(int)
+				}else if jtype == reflect.Float64{
+					r[n] = int(t.(float64))
 				}else{
-					return errors.New(fmt.Sprintf("invalid data type for %s, expect bigint but found %s", p, t))
+					return fmt.Errorf("invalid data type for %s, expect bigint but found %s", n, t)
 				}
 			case xsql.FLOAT:
 				if jtype == reflect.Float64{
-					r[p] = t.(float64)
+					r[n] = t.(float64)
 				}else{
-					return errors.New(fmt.Sprintf("invalid data type for %s, expect float but found %s", p, t))
+					return fmt.Errorf("invalid data type for %s, expect float but found %s", n, t)
 				}
 			case xsql.STRINGS:
 				if jtype == reflect.String{
-					r[p] = t.(string)
+					r[n] = t.(string)
 				}else{
-					return errors.New(fmt.Sprintf("invalid data type for %s, expect string but found %s", p, t))
+					return fmt.Errorf("invalid data type for %s, expect string but found %s", n, t)
 				}
 			case xsql.DATETIME:
-				return errors.New(fmt.Sprintf("invalid data type for %s, datetime type is not supported yet", p))
+				switch jtype {
+				case reflect.Int:
+					ai := t.(int64)
+					r[n] = common.TimeFromUnixMilli(ai)
+				case reflect.Float64:
+					ai := int64(t.(float64))
+					r[n] = common.TimeFromUnixMilli(ai)
+				case reflect.String:
+					if t, err := p.parseTime(t.(string)); err != nil{
+						return fmt.Errorf("invalid data type for %s, cannot convert to datetime: %s", n, err)
+					}else{
+						r[n] = t
+					}
+				default:
+					return fmt.Errorf("invalid data type for %s, expect datatime but find %v", n, t)
+				}
 			case xsql.BOOLEAN:
 				if jtype == reflect.Bool{
-					r[p] = t.(bool)
+					r[n] = t.(bool)
 				}else{
-					return errors.New(fmt.Sprintf("invalid data type for %s, expect boolean but found %s", p, t))
+					return fmt.Errorf("invalid data type for %s, expect boolean but found %s", n, t)
 				}
 			default:
-				return errors.New(fmt.Sprintf("invalid data type for %s, it is not supported yet", st))
+				return fmt.Errorf("invalid data type for %s, it is not supported yet", st)
 			}
 		case *xsql.ArrayType:
 			if jtype != reflect.Slice{
-				return errors.New(fmt.Sprintf("invalid data type for %s, expect array but found %s", p, t))
+				return fmt.Errorf("invalid data type for %s, expect array but found %s", n, t)
 			}
-			if tempArr, err := addArrayField(st, t.([]interface{})); err !=nil{
+			if tempArr, err := p.addArrayField(st, t.([]interface{})); err !=nil{
 				return err
 			}else {
-				r[p] = tempArr
+				r[n] = tempArr
 			}
 		case *xsql.RecType:
 			if jtype != reflect.Map{
-				return errors.New(fmt.Sprintf("invalid data type for %s, expect struct but found %s", p, t))
+				return fmt.Errorf("invalid data type for %s, expect struct but found %s", n, t)
 			}
-			nextJ, ok := j[p].(map[string]interface{})
+			nextJ, ok := j[n].(map[string]interface{})
 			if !ok {
-				return errors.New(fmt.Sprintf("invalid data type for %s, expect map but found %s", p, t))
+				return fmt.Errorf("invalid data type for %s, expect map but found %s", n, t)
 			}
 			nextR := make(map[string]interface{})
 			for _, nextF := range st.StreamFields {
-				nextP := nextF.Name
-				if e := addRecField(nextF.FieldType, nextR, nextJ, nextP); e != nil{
+				nextP := strings.ToLower(nextF.Name)
+				if e := p.addRecField(nextF.FieldType, nextR, nextJ, nextP); e != nil{
 					return e
 				}
 			}
-			r[p] = nextR
+			r[n] = nextR
 		default:
-			return errors.New(fmt.Sprintf("unsupported type %T", st))
+			return fmt.Errorf("unsupported type %T", st)
 		}
 		return nil
 	}else{
-		return errors.New(fmt.Sprintf("invalid data %s, field %s not found", j, p))
+		return fmt.Errorf("invalid data %s, field %s not found", j, n)
 	}
 }
 
 //ft must be xsql.ArrayType
 //side effect: r[p] will be set to the new array
-func addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, error) {
+func (p *Preprocessor) addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, error) {
 	if ft.FieldType != nil { //complex type array or struct
 		switch st := ft.FieldType.(type) { //Only two complex types supported here
 		case *xsql.ArrayType: //TODO handle array of array. Now the type is treated as interface{}
 			var tempSlice [][]interface{}
 			for i, t := range srcSlice{
 				if reflect.ValueOf(t).Kind() == reflect.Array{
-					if tempArr, err := addArrayField(st, t.([]interface{})); err !=nil{
+					if tempArr, err := p.addArrayField(st, t.([]interface{})); err !=nil{
 						return nil, err
 					}else {
 						tempSlice = append(tempSlice, tempArr.([]interface{}))
 					}
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect array but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect array but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
@@ -146,35 +195,35 @@ func addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, err
 				if reflect.ValueOf(t).Kind() == reflect.Map{
 					j, ok := t.(map[string]interface{})
 					if !ok {
-						return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect map but found %s", i, t))
+						return nil, fmt.Errorf("invalid data type for [%d], expect map but found %s", i, t)
 					}
 					r := make(map[string]interface{})
 					for _, f := range st.StreamFields {
-						p := f.Name
-						if e := addRecField(f.FieldType, r, j, p); e != nil{
+						n := f.Name
+						if e := p.addRecField(f.FieldType, r, j, n); e != nil{
 							return nil, e
 						}
 					}
 					tempSlice = append(tempSlice, r)
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect float but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
 		default:
-			return nil, errors.New(fmt.Sprintf("unsupported type %T", st))
+			return nil, fmt.Errorf("unsupported type %T", st)
 		}
 	}else{ //basic type
 		switch ft.Type {
 		case xsql.UNKNOWN:
-			return nil, errors.New(fmt.Sprintf("invalid data type unknown defined for %s, please checke the stream definition", srcSlice))
+			return nil, fmt.Errorf("invalid data type unknown defined for %s, please checke the stream definition", srcSlice)
 		case xsql.BIGINT:
 			var tempSlice []int
 			for i, t := range srcSlice {
 				if reflect.ValueOf(t).Kind() == reflect.Float64{
 					tempSlice = append(tempSlice, int(t.(float64)))
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect float but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
@@ -184,7 +233,7 @@ func addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, err
 				if reflect.ValueOf(t).Kind() == reflect.Float64{
 					tempSlice = append(tempSlice, t.(float64))
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect float but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect float but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
@@ -194,24 +243,44 @@ func addArrayField(ft *xsql.ArrayType, srcSlice []interface{}) (interface{}, err
 				if reflect.ValueOf(t).Kind() == reflect.String{
 					tempSlice = append(tempSlice, t.(string))
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect string but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect string but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
 		case xsql.DATETIME:
-			return nil, errors.New(fmt.Sprintf("invalid data type for %s, datetime type is not supported yet", srcSlice))
+			var tempSlice []time.Time
+			for i, t := range srcSlice {
+				jtype := reflect.ValueOf(t).Kind()
+				switch jtype {
+				case reflect.Int:
+					ai := t.(int64)
+					tempSlice = append(tempSlice, common.TimeFromUnixMilli(ai))
+				case reflect.Float64:
+					ai := int64(t.(float64))
+					tempSlice = append(tempSlice, common.TimeFromUnixMilli(ai))
+				case reflect.String:
+					if ai, err := p.parseTime(t.(string)); err != nil{
+						return nil, fmt.Errorf("invalid data type for %s, cannot convert to datetime: %s", t, err)
+					}else{
+						tempSlice = append(tempSlice, ai)
+					}
+				default:
+					return nil, fmt.Errorf("invalid data type for [%d], expect datetime but found %v", i, t)
+				}
+			}
+			return tempSlice, nil
 		case xsql.BOOLEAN:
 			var tempSlice []bool
 			for i, t := range srcSlice {
 				if reflect.ValueOf(t).Kind() == reflect.Bool{
 					tempSlice = append(tempSlice, t.(bool))
 				}else{
-					return nil, errors.New(fmt.Sprintf("invalid data type for [%d], expect boolean but found %s", i, t))
+					return nil, fmt.Errorf("invalid data type for [%d], expect boolean but found %s", i, t)
 				}
 			}
 			return tempSlice, nil
 		default:
-			return nil, errors.New(fmt.Sprintf("invalid data type for %T, datetime type is not supported yet", ft.Type))
+			return nil, fmt.Errorf("invalid data type for %T, datetime type is not supported yet", ft.Type)
 		}
 	}
 }

xsql/plans/preprocessor_test.go

@@ -1,11 +1,14 @@
 package plans
 
 import (
+	"encoding/json"
 	"engine/common"
 	"engine/xsql"
 	"fmt"
+	"log"
 	"reflect"
 	"testing"
+	"time"
 )
 
 func TestPreprocessor_Apply(t *testing.T) {
@@ -35,8 +38,9 @@ func TestPreprocessor_Apply(t *testing.T) {
 				},
 			},
 			data: []byte(`{"abc": 6}`),
-			result: map[string]interface{}{
-				"abc" : int(6),
+			result: &xsql.Tuple{Message: xsql.Message{
+					"abc": int(6),
+				},
 			},
 		},
 		{
@@ -48,9 +52,10 @@ func TestPreprocessor_Apply(t *testing.T) {
 				},
 			},
 			data: []byte(`{"abc": 34, "def" : "hello", "ghi": 50}`),
-			result: map[string]interface{}{
-				"abc" : float64(34),
-				"def" : "hello",
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": float64(34),
+				"def": "hello",
+				},
 			},
 		},
 		{
@@ -64,7 +69,6 @@ func TestPreprocessor_Apply(t *testing.T) {
 			data: []byte(`{"abc": 77, "def" : "hello"}`),
 			result: nil,
 		},
-		//Array type
 		{
 			stmt: &xsql.StreamStmt{
 				Name: xsql.StreamName("demo"),
@@ -89,13 +93,13 @@ func TestPreprocessor_Apply(t *testing.T) {
 				},
 			},
 			data: []byte(`{"a": {"b" : "hello"}}`),
-			result: map[string]interface{}{
-				"a" : map[string]interface{}{
+			result: &xsql.Tuple{Message: xsql.Message{
+					"a": map[string]interface{}{
 					"b": "hello",
 				},
 			},
+			},
 		},
-
 		//Array of complex type
 		{
 			stmt: &xsql.StreamStmt{
@@ -112,10 +116,11 @@ func TestPreprocessor_Apply(t *testing.T) {
 				},
 			},
 			data: []byte(`{"a": [{"b" : "hello1"}, {"b" : "hello2"}]}`),
-			result: map[string]interface{}{
-				"a" : []map[string]interface{}{
-					{"b": "hello1"},
-					{"b": "hello2"},
+			result: &xsql.Tuple{Message: xsql.Message{
+					"a": []map[string]interface{}{
+						{"b": "hello1"},
+						{"b": "hello2"},
+					},
 				},
 			},
 		},
@@ -137,11 +142,12 @@ func TestPreprocessor_Apply(t *testing.T) {
 				},
 			},
 			data: []byte(`{"a": {"b" : "hello", "c": {"d": 35.2}}}`),
-			result: map[string]interface{}{
-				"a" : map[string]interface{}{
-					"b" : "hello",
-					"c" : map[string]interface{}{
-						"d": int(35),
+			result: &xsql.Tuple{Message: xsql.Message{
+					"a": map[string]interface{}{
+						"b": "hello",
+						"c": map[string]interface{}{
+							"d": int(35),
+						},
 					},
 				},
 			},
@@ -153,10 +159,295 @@ func TestPreprocessor_Apply(t *testing.T) {
 	defer common.CloseLogger()
 	for i, tt := range tests {
 
-		pp := &Preprocessor{StreamStmt:tt.stmt}
-		result := pp.Apply(nil, tt.data)
-		if !reflect.DeepEqual(tt.result, result) {
-			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.data, tt.result, result)
+		pp := &Preprocessor{streamStmt: tt.stmt}
+
+		dm := make(map[string]interface{})
+		if e := json.Unmarshal(tt.data, &dm); e != nil {
+			log.Fatal(e)
+			return
+		} else {
+			tuple := &xsql.Tuple{Message:dm}
+			result := pp.Apply(nil, tuple)
+			if !reflect.DeepEqual(tt.result, result) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tuple, tt.result, result)
+			}
 		}
+
+	}
+}
+
+func TestPreprocessorTime_Apply(t *testing.T){
+	var tests = []struct {
+		stmt *xsql.StreamStmt
+		data []byte
+		result interface{}
+	}{
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+				},
+			},
+			data: []byte(`{"abc": "2019-09-19T00:55:15.000Z", "def" : 1568854573431}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": common.TimeFromUnixMilli(1568854515000),
+				"def": common.TimeFromUnixMilli(1568854573431),
+			},
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+				},
+			},
+			data: []byte(`{"abc": "2019-09-19T00:55:1dd5Z", "def" : 111568854573431}`),
+			result: nil,
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+					"CONF_KEY" : "srv1",
+					"TYPE" : "MQTT",
+					"TIMESTAMP" : "USERID",
+					"TIMESTAMP_FORMAT" : "yyyy-MM-dd 'at' HH:mm:ss'Z'X",
+				},
+			},
+			data: []byte(`{"abc": "2019-09-19 at 18:55:15Z+07", "def" : 1568854573431}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": common.TimeFromUnixMilli(1568894115000),
+				"def": common.TimeFromUnixMilli(1568854573431),
+			}},
+		},
+		//Array type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "a", FieldType: &xsql.ArrayType{
+						Type: xsql.DATETIME,
+					}},
+				},
+			},
+			data: []byte(`{"a": [1568854515123, 1568854573431]}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"a": []time.Time{
+					common.TimeFromUnixMilli(1568854515123),
+					common.TimeFromUnixMilli(1568854573431),
+				},
+			},
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "a", FieldType: &xsql.RecType{
+						StreamFields: []xsql.StreamField{
+							{Name: "b", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+							{Name: "c", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+						},
+					}},
+				},
+			},
+			data: []byte(`{"a": {"b" : "hello", "c": 1568854515000}}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"a": map[string]interface{}{
+					"b": "hello",
+					"c": common.TimeFromUnixMilli(1568854515000),
+				},
+			},
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+
+	defer common.CloseLogger()
+	for i, tt := range tests {
+
+		pp := &Preprocessor{streamStmt: tt.stmt}
+
+		dm := make(map[string]interface{})
+		if e := json.Unmarshal(tt.data, &dm); e != nil {
+			log.Fatal(e)
+			return
+		} else {
+			tuple := &xsql.Tuple{Message:dm}
+			result := pp.Apply(nil, tuple)
+			//workaround make sure all the timezone are the same for time vars or the DeepEqual will be false.
+			if rt, ok := result.(*xsql.Tuple); ok{
+				if rtt, ok := rt.Message["abc"].(time.Time); ok{
+					rt.Message["abc"] = rtt.UTC()
+				}
+			}
+			if !reflect.DeepEqual(tt.result, result) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tuple, tt.result, result)
+			}
+		}
+
+	}
+}
+
+func TestPreprocessorEventtime_Apply(t *testing.T) {
+
+	var tests = []struct {
+		stmt *xsql.StreamStmt
+		data []byte
+		result interface{}
+	}{
+		//Basic type
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.BIGINT}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"FORMAT" : "AVRO",
+					"KEY" : "USERID",
+					"CONF_KEY" : "srv1",
+					"TYPE" : "MQTT",
+					"TIMESTAMP" : "abc",
+					"TIMESTAMP_FORMAT" : "yyyy-MM-dd''T''HH:mm:ssX'",
+				},
+			},
+			data: []byte(`{"abc": 1568854515000}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": int(1568854515000),
+			}, Timestamp: 1568854515000,
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.BOOLEAN}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"TIMESTAMP" : "abc",
+				},
+			},
+			data: []byte(`{"abc": true}`),
+			result: nil,
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"TIMESTAMP" : "def",
+				},
+			},
+			data: []byte(`{"abc": 34, "def" : "2019-09-23T02:47:29.754Z", "ghi": 50}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": float64(34),
+				"def": "2019-09-23T02:47:29.754Z",
+			}, Timestamp: int64(1569206849754),
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.DATETIME}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"TIMESTAMP" : "abc",
+				},
+			},
+			data: []byte(`{"abc": "2019-09-19T00:55:15.000Z", "def" : 1568854573431}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": common.TimeFromUnixMilli(1568854515000),
+				"def": common.TimeFromUnixMilli(1568854573431),
+			}, Timestamp: int64(1568854515000),
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"TIMESTAMP" : "def",
+					"TIMESTAMP_FORMAT" : "yyyy-MM-dd'AT'HH:mm:ss",
+				},
+			},
+			data: []byte(`{"abc": 34, "def" : "2019-09-23AT02:47:29", "ghi": 50}`),
+			result: &xsql.Tuple{Message: xsql.Message{
+				"abc": float64(34),
+				"def": "2019-09-23AT02:47:29",
+			}, Timestamp: int64(1569206849000),
+			},
+		},
+		{
+			stmt: &xsql.StreamStmt{
+				Name: xsql.StreamName("demo"),
+				StreamFields: []xsql.StreamField{
+					{Name: "abc", FieldType: &xsql.BasicType{Type: xsql.FLOAT}},
+					{Name: "def", FieldType: &xsql.BasicType{Type: xsql.STRINGS}},
+				},
+				Options: map[string]string{
+					"DATASOURCE" : "users",
+					"TIMESTAMP" : "def",
+					"TIMESTAMP_FORMAT" : "yyyy-MM-ddaHH:mm:ss",
+				},
+			},
+			data: []byte(`{"abc": 34, "def" : "2019-09-23AT02:47:29", "ghi": 50}`),
+			result: nil,
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+
+	defer common.CloseLogger()
+	for i, tt := range tests {
+
+		pp, err := NewPreprocessor(tt.stmt, true)
+		if err != nil{
+			t.Error(err)
+		}
+
+		dm := make(map[string]interface{})
+		if e := json.Unmarshal(tt.data, &dm); e != nil {
+			log.Fatal(e)
+			return
+		} else {
+			tuple := &xsql.Tuple{Message:dm}
+			result := pp.Apply(nil, tuple)
+			//workaround make sure all the timezone are the same for time vars or the DeepEqual will be false.
+			if rt, ok := result.(*xsql.Tuple); ok{
+				if rtt, ok := rt.Message["abc"].(time.Time); ok{
+					rt.Message["abc"] = rtt.UTC()
+				}
+			}
+			if !reflect.DeepEqual(tt.result, result) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tuple, tt.result, result)
+			}
+		}
+
 	}
 }

xsql/plans/project_operator.go

@@ -3,6 +3,7 @@ package plans
 import (
 	"context"
 	"encoding/json"
+	"engine/common"
 	"engine/xsql"
 	"fmt"
 	"strconv"
@@ -11,39 +12,103 @@ import (
 
 type ProjectPlan struct {
 	Fields xsql.Fields
+	IsAggregate bool
 }
 
+/**
+ *  input: *xsql.Tuple from preprocessor or filterOp | xsql.WindowTuplesSet from windowOp or filterOp | xsql.JoinTupleSets from joinOp or filterOp
+ *  output: []map[string]interface{}
+ */
 func (pp *ProjectPlan) Apply(ctx context.Context, data interface{}) interface{} {
-	var input map[string]interface{}
-	if d, ok := data.(map[string]interface{}); !ok {
-		fmt.Printf("Expect map[string]interface{} type.\n")
+	log := common.GetLogger(ctx)
+	log.Debugf("project plan receive %s", data)
+	var results []map[string]interface{}
+	switch input := data.(type) {
+	case *xsql.Tuple:
+		ve := pp.getVE(input, input)
+		results = append(results, project(pp.Fields, ve))
+	case xsql.WindowTuplesSet:
+		if len(input) != 1 {
+			log.Infof("WindowTuplesSet with multiple tuples cannot be evaluated")
+			return nil
+		}
+		ms := input[0].Tuples
+		for _, v := range ms {
+			ve := pp.getVE(&v, input)
+			results = append(results, project(pp.Fields, ve))
+			if pp.IsAggregate{
+				break
+			}
+		}
+	case xsql.JoinTupleSets:
+		ms := input
+		for _, v := range ms {
+			ve := pp.getVE(&v, input)
+			results = append(results, project(pp.Fields, ve))
+			if pp.IsAggregate{
+				break
+			}
+		}
+	case xsql.GroupedTuplesSet:
+		for _, v := range input{
+			ve := pp.getVE(v[0], v)
+			results = append(results, project(pp.Fields, ve))
+		}
+	default:
+		log.Errorf("Expect xsql.Valuer or its array type")
 		return nil
-	} else {
-		input = d
 	}
 
-	var result = make(map[string]interface{})
+	if ret, err := json.Marshal(results); err == nil {
+		return ret
+	} else {
+		fmt.Printf("Found error: %v", err)
+		return nil
+	}
+}
 
-	ve := &xsql.ValuerEval{Valuer: xsql.MultiValuer(xsql.MapValuer(input), &xsql.FunctionValuer{}, &xsql.WildcardValuer{Data: input})}
+func (pp *ProjectPlan) getVE(tuple xsql.DataValuer, agg xsql.AggregateData) *xsql.ValuerEval{
+	if pp.IsAggregate{
+		return &xsql.ValuerEval{Valuer: xsql.MultiAggregateValuer(agg, tuple, &xsql.FunctionValuer{}, &xsql.AggregateFunctionValuer{Data: agg}, &xsql.WildcardValuer{Data: tuple})}
+	}else{
+		return &xsql.ValuerEval{Valuer: xsql.MultiValuer(tuple, &xsql.FunctionValuer{}, &xsql.WildcardValuer{Data: tuple})}
+	}
+}
 
-	for _, f := range pp.Fields {
+func project(fs xsql.Fields, ve *xsql.ValuerEval) map[string]interface{} {
+	result := make(map[string]interface{})
+	for _, f := range fs {
 		v := ve.Eval(f.Expr)
-		if val, ok := v.(map[string]interface{}); ok { //It should be the asterisk in fields list.
-			result = val
-			break
+		if _, ok := f.Expr.(*xsql.Wildcard); ok || f.Name == "*"{
+			switch val := v.(type) {
+			case map[string]interface{} :
+				for k, v := range val{
+					if _, ok := result[k]; !ok{
+						result[k] = v
+					}
+				}
+			case xsql.Message:
+				for k, v := range val{
+					if _, ok := result[k]; !ok{
+						result[k] = v
+					}
+				}
+			default:
+				fmt.Printf("Wildcarder does not return map")
+			}
 		} else {
-			result[assignName(f.Name, f.AName, result)] = v
+			if v != nil {
+				n := assignName(f.Name, f.AName, result)
+				if _, ok := result[n]; !ok{
+					result[n] = v
+				}
+			}
 		}
 	}
-
-	if ret, err := json.Marshal(result); err == nil {
-		return ret
-	} else {
-		fmt.Printf("Found error: %v.\n", err)
-		return nil
-	}
+	return result
 }
 
+
 const DEFAULT_FIELD_NAME_PREFIX string = "rengine_field_"
 
 func assignName(name, alias string, fields map[string] interface{}) string {
@@ -61,6 +126,6 @@ func assignName(name, alias string, fields map[string] interface{}) string {
 			return key
 		}
 	}
-	fmt.Printf("Cannot assign a default field name.\n")
+	fmt.Printf("Cannot assign a default field name")
 	return ""
 }

xsql/plans/str_func_test.go

@@ -0,0 +1,382 @@
+package plans
+
+import (
+	"encoding/json"
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestStrFunc_Apply1(t *testing.T) {
+	var tests = []struct {
+		sql  string
+		data *xsql.Tuple
+		result []map[string]interface{}
+	}{
+		{
+			sql: "SELECT concat(a, b, c) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "myamybmyc",
+			}},
+		},
+
+		{
+			sql: "SELECT endswith(a, b) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "myb",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": false,
+			}},
+		},
+		{
+			sql: "SELECT endswith(a, b) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": true,
+			}},
+		},
+		{
+			sql: "SELECT format_time(a, \"yyyy-MM-dd T HH:mm:ss\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : common.TimeFromUnixMilli(1568854515000),
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "2019-09-19 T 00:55:15",
+			}},
+		},
+		{
+			sql: "SELECT indexof(a, \"a\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(2),
+			}},
+		},
+		{
+			sql: "SELECT length(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "中国",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(2),
+			}},
+		},
+		{
+			sql: "SELECT length(c) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "中国",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(3),
+			}},
+		},
+		{
+			sql: "SELECT lower(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "nycnicks",
+			}},
+		},
+		{
+			sql: "SELECT lpad(a, 2) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "  NYCNicks",
+			}},
+		},
+		{
+			sql: "SELECT ltrim(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : " \ttrimme\n ",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "trimme\n ",
+			}},
+		},
+		{
+			sql: "SELECT numbytes(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "中国",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(6),
+			}},
+		},
+		{
+			sql: "SELECT numbytes(b) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "中国",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": float64(2),
+			}},
+		},
+		{
+			sql: "SELECT regexp_matches(a,\"foo.*\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "seafood",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": true,
+			}},
+		},
+		{
+			sql: "SELECT regexp_matches(b,\"foo.*\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "seafood",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": false,
+			}},
+		},
+		{
+			sql: "SELECT regexp_replace(a,\"a(x*)b\", \"REP\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "-ab-axxb-",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "-REP-REP-",
+			}},
+		},
+		{
+			sql: "SELECT regexp_substr(a,\"foo.*\") AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "seafood",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "food",
+			}},
+		},
+		{
+			sql: "SELECT rpad(a, 3) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "NYCNicks   ",
+			}},
+		},
+		{
+			sql: "SELECT rtrim(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : " \ttrimme\n ",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": " \ttrimme",
+			}},
+		},
+		{
+			sql: "SELECT substring(a, 3) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "Nicks",
+			}},
+		},
+		{
+			sql: "SELECT substring(a, 3, 5) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "Ni",
+			}},
+		},
+		{
+			sql: "SELECT endswith(a, b) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": true,
+			}},
+		},
+		{
+			sql: "SELECT endswith(a, c) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "mya",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": false,
+			}},
+		},
+		{
+			sql: "SELECT trim(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : " \ttrimme\n ",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "trimme",
+			}},
+		},
+		{
+			sql: "SELECT upper(a) AS a FROM test",
+			data: &xsql.Tuple{
+				Emitter: "test",
+				Message: xsql.Message{
+					"a" : "NYCNicks",
+					"b" : "ya",
+					"c" : "myc",
+				},
+			},
+			result: []map[string]interface{}{{
+				"a": "NYCNICKS",
+			}},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		stmt, err := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
+		if err != nil || stmt == nil {
+			t.Errorf("parse sql %s error %v", tt.sql, err)
+		}
+		pp := &ProjectPlan{Fields:stmt.Fields}
+		result := pp.Apply(nil, tt.data)
+		var mapRes []map[string]interface{}
+		if v, ok := result.([]byte); ok {
+			err := json.Unmarshal(v, &mapRes)
+			if err != nil {
+				t.Errorf("Failed to parse the input into map.\n")
+				continue
+			}
+			//fmt.Printf("%t\n", mapRes["rengine_field_0"])
+
+			if !reflect.DeepEqual(tt.result, mapRes) {
+				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, mapRes)
+			}
+		} else {
+			t.Errorf("The returned result is not type of []byte\n")
+		}
+	}
+}

xsql/processors/xsql_processor.go

@@ -2,14 +2,17 @@ package processors
 
 import (
 	"bytes"
+	"encoding/json"
 	"engine/common"
 	"engine/xsql"
 	"engine/xsql/plans"
 	"engine/xstream"
-	"engine/xstream/collectors"
 	"engine/xstream/extensions"
+	"engine/xstream/operators"
+	"engine/xstream/sinks"
 	"fmt"
 	"github.com/dgraph-io/badger"
+	"path"
 	"strings"
 )
 
@@ -127,7 +130,7 @@ func (p *StreamProcessor) execDropStream(stmt *xsql.DropStreamStatement, db *bad
 }
 
 func GetStream(db *badger.DB, name string) (stmt *xsql.StreamStmt, err error){
-	s, err := common.DbGet(db, string(name))
+	s, err := common.DbGet(db, name)
 	if err != nil {
 		return
 	}
@@ -143,84 +146,257 @@ func GetStream(db *badger.DB, name string) (stmt *xsql.StreamStmt, err error){
 
 
 type RuleProcessor struct {
-	sql string
-//	actions string
 	badgerDir string
 }
 
-func NewRuleProcessor(s, d string) *RuleProcessor {
+func NewRuleProcessor(d string) *RuleProcessor {
 	processor := &RuleProcessor{
-		sql: s,
 		badgerDir: d,
 	}
 	return processor
 }
 
-func (p *RuleProcessor) Exec() error {
-	parser := xsql.NewParser(strings.NewReader(p.sql))
-	if stmt, err := xsql.Language.Parse(parser); err != nil{
-		return fmt.Errorf("parse sql %s error: %s", p.sql , err)
+func (p *RuleProcessor) ExecCreate(name, ruleJson string) (*xstream.Rule, error) {
+	rule, err := p.getRuleByJson(name, ruleJson)
+	if err != nil {
+		return nil, err
+	}
+	db, err := common.DbOpen(path.Join(p.badgerDir, "rule"))
+	if err != nil {
+		return nil, err
+	}
+	err = common.DbSet(db, string(name), ruleJson)
+	if err != nil {
+		common.DbClose(db)
+		return nil, err
+	}else{
+		log.Infof("rule %s created", name)
+		common.DbClose(db)
+	}
+	return rule, nil
+}
+
+func (p *RuleProcessor) GetRuleByName(name string) (*xstream.Rule, error) {
+	db, err := common.DbOpen(path.Join(p.badgerDir, "rule"))
+	if err != nil {
+		return nil, err
+	}
+	defer common.DbClose(db)
+	s, err := common.DbGet(db, string(name))
+	if err != nil {
+		return nil, fmt.Errorf("rule %s not found", name)
+	}
+	return p.getRuleByJson(name, s)
+}
+
+func (p *RuleProcessor) getRuleByJson(name, ruleJson string) (*xstream.Rule, error) {
+	var rule xstream.Rule
+	if err := json.Unmarshal([]byte(ruleJson), &rule); err != nil {
+		return nil, fmt.Errorf("parse rule %s error : %s", ruleJson, err)
+	}
+	rule.Id = name
+	//validation
+	if name == ""{
+		return nil, fmt.Errorf("missing rule id")
+	}
+	if rule.Sql == ""{
+		return nil, fmt.Errorf("missing rule sql")
+	}
+	if rule.Actions == nil || len(rule.Actions) == 0{
+		return nil, fmt.Errorf("missing rule actions")
+	}
+	return &rule, nil
+}
+
+func (p *RuleProcessor) ExecInitRule(rule *xstream.Rule) (*xstream.TopologyNew, error) {
+	if tp, inputs, err := p.createTopo(rule); err != nil {
+		return nil, err
+	}else{
+		for _, m := range rule.Actions {
+			for name, action := range m {
+				switch name {
+				case "log":
+					log.Printf("Create log sink with %s", action)
+					tp.AddSink(inputs, sinks.NewLogSink("sink_log", rule.Id))
+				case "mqtt":
+					log.Printf("Create mqtt sink with %s", action)
+					if ms, err := sinks.NewMqttSink("mqtt_log", rule.Id, action); err != nil{
+						return nil, err
+					}else{
+						tp.AddSink(inputs, ms)
+					}
+				default:
+					return nil, fmt.Errorf("unsupported action: %s", name)
+				}
+			}
+		}
+		return tp, nil
+	}
+}
+
+func (p *RuleProcessor) ExecQuery(ruleid, sql string) (*xstream.TopologyNew, error) {
+	if tp, inputs, err := p.createTopo(&xstream.Rule{Id: ruleid, Sql: sql}); err != nil {
+		return nil, err
+	} else {
+		tp.AddSink(inputs, sinks.NewLogSinkToMemory("sink_log", ruleid))
+		go func() {
+			select {
+			case err := <-tp.Open():
+				log.Println(err)
+				tp.Cancel()
+			}
+		}()
+		return tp, nil
+	}
+}
+
+func (p *RuleProcessor) ExecDesc(name string) (string, error) {
+	db, err := common.DbOpen(path.Join(p.badgerDir, "rule"))
+	if err != nil {
+		return "", err
+	}
+	defer common.DbClose(db)
+	s, err := common.DbGet(db, string(name))
+	if err != nil {
+		return "", fmt.Errorf("rule %s not found", name)
+	}
+	dst := &bytes.Buffer{}
+	if err := json.Indent(dst, []byte(s), "", "  "); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintln(dst.String()), nil
+}
+
+func (p *RuleProcessor) ExecShow() (string, error) {
+	keys, err := p.GetAllRules()
+	if err != nil{
+		return "", err
+	}
+	if len(keys) == 0 {
+		keys = append(keys, "no rule definition found")
+	}
+	var result string
+	for _, c := range keys{
+		result = result + fmt.Sprintln(c)
+	}
+	return result, nil
+}
+
+func (p *RuleProcessor) GetAllRules() ([]string, error) {
+	db, err := common.DbOpen(path.Join(p.badgerDir, "rule"))
+	if err != nil {
+		return nil, err
+	}
+	defer common.DbClose(db)
+	return common.DbKeys(db)
+}
+
+func (p *RuleProcessor) ExecDrop(name string) (string, error) {
+	db, err := common.DbOpen(path.Join(p.badgerDir, "rule"))
+	if err != nil {
+		return "", err
+	}
+	defer common.DbClose(db)
+	err = common.DbDelete(db, string(name))
+	if err != nil {
+		return "", err
 	}else{
-		if selectStmt, ok := stmt.(*xsql.SelectStatement); !ok{
-			return fmt.Errorf("sql %s is not a select statement", p.sql)
-		}else{
-			//TODO Validation here or in the cli?
-			tp := xstream.New()
-
-			//create sources and preprocessor
-			db, err := common.DbOpen(p.badgerDir)
+		return fmt.Sprintf("rule %s dropped", name), nil
+	}
+}
+
+func (p *RuleProcessor) createTopo(rule *xstream.Rule) (*xstream.TopologyNew, []xstream.Emitter, error) {
+	return p.createTopoWithSources(rule, nil)
+}
+
+//For test to mock source
+func (p *RuleProcessor) createTopoWithSources(rule *xstream.Rule, sources []xstream.Source) (*xstream.TopologyNew, []xstream.Emitter, error){
+	name := rule.Id
+	sql := rule.Sql
+	var isEventTime bool
+	var lateTol int64
+	if iet, ok := rule.Options["isEventTime"]; ok{
+		isEventTime, ok = iet.(bool)
+		if !ok{
+			return nil, nil, fmt.Errorf("invalid rule option isEventTime %v, bool type required", iet)
+		}
+	}
+	if isEventTime {
+		if l, ok := rule.Options["lateTolerance"]; ok{
+			if fl, ok := l.(float64); ok{
+				lateTol = int64(fl)
+			}else{
+				return nil, nil, fmt.Errorf("invalid rule option lateTolerance %v, int type required", l)
+			}
+		}
+	}
+	shouldCreateSource := sources == nil
+	parser := xsql.NewParser(strings.NewReader(sql))
+	if stmt, err := xsql.Language.Parse(parser); err != nil{
+		return nil, nil, fmt.Errorf("parse sql %s error: %s", sql , err)
+	}else {
+		if selectStmt, ok := stmt.(*xsql.SelectStatement); !ok {
+			return nil, nil, fmt.Errorf("sql %s is not a select statement", sql)
+		} else {
+			tp := xstream.NewWithName(name)
+			var inputs []xstream.Emitter
+			streamsFromStmt := xsql.GetStreams(selectStmt)
+			if !shouldCreateSource && len(streamsFromStmt) != len(sources){
+				return nil, nil, fmt.Errorf("invalid parameter sources or streams, the length cannot match the statement, expect %d sources", len(streamsFromStmt))
+			}
+			db, err := common.DbOpen(path.Join(p.badgerDir, "stream"))
 			if err != nil {
-				return err
+				return nil, nil, err
 			}
 			defer common.DbClose(db)
-			var inputs []xstream.Emitter
-			for _, s := range selectStmt.Sources {
-				switch t := s.(type){
-				case *xsql.Table:
-					if streamStmt, err := GetStream(db, t.Name); err != nil{
-						return err
-					} else {
-						mqs, err := extensions.NewWithName(string(streamStmt.Name), streamStmt.Options["DATASOURCE"], streamStmt.Options["CONF_KEY"])
-						if err != nil {
-							return err
-						}
-						tp.AddSrc(mqs)
-
-						preprocessorOp := xstream.Transform(&plans.Preprocessor{StreamStmt: streamStmt}, "preprocessor_" +t.Name)
-						tp.AddOperator([]xstream.Emitter{mqs}, preprocessorOp)
-						inputs = append(inputs, preprocessorOp)
+
+			for i, s := range streamsFromStmt {
+				streamStmt, err := GetStream(db, s)
+				if err != nil {
+					return nil, nil, err
+				}
+				pp, err := plans.NewPreprocessor(streamStmt, isEventTime)
+				if err != nil{
+					return nil, nil, err
+				}
+				if shouldCreateSource{
+					mqs, err := extensions.NewWithName(string(streamStmt.Name), streamStmt.Options["DATASOURCE"], streamStmt.Options["CONF_KEY"])
+					if err != nil {
+						return nil, nil, err
 					}
-				default:
-					return fmt.Errorf("unsupported source type %T", s)
+					tp.AddSrc(mqs)
+					preprocessorOp := xstream.Transform(pp, "preprocessor_"+s)
+					tp.AddOperator([]xstream.Emitter{mqs}, preprocessorOp)
+					inputs = append(inputs, preprocessorOp)
+				}else{
+					tp.AddSrc(sources[i])
+					preprocessorOp := xstream.Transform(pp, "preprocessor_"+s)
+					tp.AddOperator([]xstream.Emitter{sources[i]}, preprocessorOp)
+					inputs = append(inputs, preprocessorOp)
+				}
+			}
+			dimensions := selectStmt.Dimensions
+			var w *xsql.Window
+			if dimensions != nil {
+				w = dimensions.GetWindow()
+				if w != nil {
+					wop, err := operators.NewWindowOp("window", w, isEventTime, lateTol, streamsFromStmt)
+					if err != nil {
+						return nil, nil, err
+					}
+					tp.AddOperator(inputs, wop)
+					inputs = []xstream.Emitter{wop}
 				}
 			}
 
-			//if selectStmt.Joins != nil {
-			//	for _, join := range selectStmt.Joins {
-			//		if streamStmt, err := GetStream(db, join.Name); err != nil{
-			//			return err
-			//		} else {
-			//			mqs, err := extensions.NewWithName(string(streamStmt.Name), streamStmt.Options["DATASOURCE"], streamStmt.Options["CONF_KEY"])
-			//			if err != nil {
-			//				return err
-			//			}
-			//			tp.AddSrc(mqs)
-			//
-			//			preprocessorOp := xstream.Transform(&plans.Preprocessor{StreamStmt: streamStmt}, "preprocessor_" + join.Name)
-			//			tp.AddOperator([]xstream.Emitter{mqs}, preprocessorOp)
-			//			inputs = append(inputs, preprocessorOp)
-			//		}
-			//	}
-			//
-			//	joinOp := xstream.Transform(&plans.JoinPlan{Joins:selectStmt.Joins, Dimensions: selectStmt.Dimensions}, "join")
-			//	//TODO concurrency setting by command
-			//	//joinOp.SetConcurrency(3)
-			//	//TODO Read the ticker from dimension statement
-			//	joinOp.SetTicker(time.Second * 5)
-			//	tp.AddOperator(inputs, joinOp)
-			//	inputs = []xstream.Emitter{joinOp}
-			//}
-
+			if w != nil && selectStmt.Joins != nil {
+				joinOp := xstream.Transform(&plans.JoinPlan{Joins: selectStmt.Joins, From: selectStmt.Sources[0].(*xsql.Table)}, "join")
+				//TODO concurrency setting by command
+				//joinOp.SetConcurrency(3)
+				tp.AddOperator(inputs, joinOp)
+				inputs = []xstream.Emitter{joinOp}
+			}
 
 			if selectStmt.Condition != nil {
 				filterOp := xstream.Transform(&plans.FilterPlan{Condition: selectStmt.Condition}, "filter")
@@ -230,24 +406,29 @@ func (p *RuleProcessor) Exec() error {
 				inputs = []xstream.Emitter{filterOp}
 			}
 
-			if selectStmt.Fields != nil {
-				projectOp := xstream.Transform(&plans.ProjectPlan{Fields: selectStmt.Fields}, "project")
-				tp.AddOperator(inputs, projectOp)
-				inputs = []xstream.Emitter{projectOp}
+			var ds xsql.Dimensions
+			if dimensions != nil {
+				ds = dimensions.GetGroups()
+				if ds != nil && len(ds) > 0 {
+					aggregateOp := xstream.Transform(&plans.AggregatePlan{Dimensions: ds}, "aggregate")
+					tp.AddOperator(inputs, aggregateOp)
+					inputs = []xstream.Emitter{aggregateOp}
+				}
 			}
 
+			if selectStmt.SortFields != nil {
+				orderOp := xstream.Transform(&plans.OrderPlan{SortFields:selectStmt.SortFields}, "order")
+				tp.AddOperator(inputs, orderOp)
+				inputs = []xstream.Emitter{orderOp}
+			}
 
-			//TODO hard coded sink now. parameterize it
-			tp.AddSink(inputs, collectors.Func(func(data interface{}) error {
-				fmt.Printf("Sink: %s\n", data)
-				return nil
-			}))
-
-			if err := <-tp.Open(); err != nil {
-				return err
+			if selectStmt.Fields != nil {
+				projectOp := xstream.Transform(&plans.ProjectPlan{Fields: selectStmt.Fields, IsAggregate: xsql.IsAggStatement(selectStmt)}, "project")
+				tp.AddOperator(inputs, projectOp)
+				inputs = []xstream.Emitter{projectOp}
 			}
+			return tp, inputs, nil
 		}
 	}
-	return nil
 }
 

xsql/util.go

@@ -1,6 +1,9 @@
 package xsql
 
-import "bytes"
+import (
+	"bytes"
+	"strings"
+)
 
 func PrintFieldType(ft FieldType, buff *bytes.Buffer) {
 	switch t := ft.(type) {
@@ -28,4 +31,32 @@ func PrintFieldType(ft FieldType, buff *bytes.Buffer) {
 		}
 		buff.WriteString(")")
 	}
+}
+
+func GetStreams(stmt *SelectStatement) (result []string){
+	if stmt == nil {
+		return nil
+	}
+	for _, source := range stmt.Sources{
+		if s, ok := source.(*Table); ok {
+			result = append(result, s.Name)
+		}
+	}
+
+	for _, join := range stmt.Joins{
+		result = append(result, join.Name)
+	}
+	return
+}
+
+func LowercaseKeyMap(m map[string]interface{}) (map[string]interface{}) {
+	m1 := make(map[string]interface{})
+	for k, v := range m {
+		if m2, ok := v.(map[string]interface{}); ok {
+			m1[strings.ToLower(k)] = LowercaseKeyMap(m2)
+		} else {
+			m1[strings.ToLower(k)] = v
+		}
+	}
+	return m1
 }

xsql/util_test.go

@@ -0,0 +1,70 @@
+package xsql
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestLowercaseKeyMap(t *testing.T) {
+	var tests = []struct {
+		src  map[string]interface{}
+		dest map[string]interface{}
+	}{
+		{
+			src: map[string]interface{}{
+				"Key1": "value1",
+				"key2": "value2",
+			},
+			dest: map[string]interface{}{
+				"key1": "value1",
+				"key2": "value2",
+			},
+		},
+
+		{
+			src: map[string]interface{}{
+				"Key1": "value1",
+				"Complex": map[string]interface{}{
+					"Sub1": "sub_value1",
+				},
+			},
+			dest: map[string]interface{}{
+				"key1": "value1",
+				"complex": map[string]interface{}{
+					"sub1": "sub_value1",
+				},
+			},
+		},
+
+		{
+			src: map[string]interface{}{
+				"Key1": "value1",
+				"Complex": map[string]interface{}{
+					"Sub1": "sub_value1",
+					"Sub1_2": map[string]interface{}{
+						"Sub2": "sub2",
+					},
+				},
+			},
+			dest: map[string]interface{}{
+				"key1": "value1",
+				"complex": map[string]interface{}{
+					"sub1": "sub_value1",
+					"sub1_2": map[string]interface{}{
+						"sub2": "sub2",
+					},
+				},
+			},
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		//fmt.Printf("Parsing SQL %q.\n", tt.s)
+		result := LowercaseKeyMap(tt.src)
+		if !reflect.DeepEqual(tt.dest, result) {
+			t.Errorf("%d. \nstmt mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.dest, result)
+		}
+	}
+}

xsql/xsql_stream_test.go

@@ -21,7 +21,7 @@ func TestParser_ParseCreateStream(t *testing.T) {
 					NICKNAMES ARRAY(STRING),
 					Gender BOOLEAN,
 					ADDRESS STRUCT(STREET_NAME STRING, NUMBER BIGINT),
-				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID", CONF_KEY="srv1", type="MQTT");`,
+				) WITH (DATASOURCE="users", FORMAT="AVRO", KEY="USERID", CONF_KEY="srv1", type="MQTT", TIMESTAMP="USERID", TIMESTAMP_FORMAT="yyyy-MM-dd''T''HH:mm:ssX'");`,
 			stmt: &StreamStmt{
 				Name: StreamName("demo"),
 				StreamFields: []StreamField{
@@ -43,6 +43,8 @@ func TestParser_ParseCreateStream(t *testing.T) {
 					"KEY" : "USERID",
 					"CONF_KEY" : "srv1",
 					"TYPE" : "MQTT",
+					"TIMESTAMP" : "USERID",
+					"TIMESTAMP_FORMAT" : "yyyy-MM-dd''T''HH:mm:ssX'",
 				},
 			},
 		},

xstream/cli/main.go

@@ -3,16 +3,38 @@ package main
 import (
 	"bufio"
 	"engine/common"
-	"engine/xsql/processors"
 	"fmt"
+	"github.com/go-yaml/yaml"
 	"github.com/urfave/cli"
+	"io/ioutil"
+	"net/rpc"
 	"os"
 	"sort"
 	"strings"
+	"time"
 )
 
-var log = common.Log
+type clientConf struct {
+	Host string `yaml:"host"`
+	Port int `yaml:"port"`
+}
 
+var clientYaml = "client.yaml"
+
+func streamProcess(client *rpc.Client, args string) error {
+	var reply string
+	if args == ""{
+		args = strings.Join(os.Args[1:], " ")
+	}
+	err := client.Call("Server.Stream", args, &reply)
+	if err != nil{
+		fmt.Println(err)
+		return err
+	}else{
+		fmt.Println(reply)
+	}
+	return nil
+}
 
 func main() {
 	app := cli.NewApp()
@@ -23,19 +45,44 @@ func main() {
 	//		Usage: "the name of stream",
 	//	}}
 
-	dataDir, err := common.GetDataLoc()
+	b := common.LoadConf(clientYaml)
+	var cfg map[string]clientConf
+	var config *clientConf
+	if err := yaml.Unmarshal(b, &cfg); err != nil {
+		fmt.Printf("Failed to load config file with error %s.\n", err)
+	}else{
+		c, ok := cfg["basic"]
+		if !ok{
+			fmt.Printf("No basic config in client.yaml, will use the default configuration.\n")
+		}else{
+			config = &c
+		}
+	}
+	if config == nil {
+		config = &clientConf{
+			Host: "127.0.0.1",
+			Port: 20498,
+		}
+	}
+
+	fmt.Printf("Connecting to %s:%d \n", config.Host, config.Port)
+	// Create a TCP connection to localhost on port 1234
+	client, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", config.Host, config.Port))
 	if err != nil {
-		log.Panic(err)
+		fmt.Printf("Failed to connect the server, please start the server.\n")
+		return
 	}
 
 	app.Commands = []cli.Command{
 		{
-			Name:      "stream",
-			Aliases:   []string{"s"},
-			Usage:     "manage streams",
+			Name:      "query",
+			Aliases:   []string{"query"},
+			Usage:     "query command line",
 			Action:    func(c *cli.Context) error {
 				reader := bufio.NewReader(os.Stdin)
 				var inputs []string
+				ticker := time.NewTicker(time.Millisecond * 300)
+				defer ticker.Stop()
 				for {
 					fmt.Print("xstream > ")
 
@@ -46,50 +93,335 @@ func main() {
 
 					if strings.ToLower(text) == "quit" || strings.ToLower(text) == "exit" {
 						break
+					} else if strings.Trim(text, " ") == "" {
+						continue
 					} else {
-						content, err := processors.NewStreamProcessor(text, dataDir).Exec()
-						if err != nil {
-							fmt.Printf("stream command error: %s\n", err)
-						}else{
-							for _, c := range content{
-								fmt.Println(c)
-							}
+						var reply string
+						err := client.Call("Server.CreateQuery", text, &reply)
+						if err != nil{
+							fmt.Println(err)
+							return err
+						} else {
+							fmt.Println(reply)
+							go func() {
+								for {
+									<-ticker.C
+									var result string
+									_ = client.Call("Server.GetQueryResult", "", &result)
+									if result != "" {
+										fmt.Println(result)
+									}
+								}
+							}()
 						}
 					}
 				}
 				return nil
 			},
 		},
+		{
+			Name:      "create",
+			Aliases:   []string{"create"},
+			Usage:     "create stream $stream_name | create stream $stream_name -f $stream_def_file | create rule $rule_name $rule_json | create rule $rule_name -f $rule_def_file",
 
+			Subcommands: []cli.Command {
+				{
+					Name:  "stream",
+					Usage: "create stream $stream_name [-f stream_def_file]",
+					Flags: []cli.Flag {
+						cli.StringFlag{
+							Name: "file, f",
+							Usage: "the location of stream definition file",
+							FilePath: "/home/mystream.txt",
+						},
+					},
+					Action: func(c *cli.Context) error {
+						sfile := c.String("file")
+						if sfile != "" {
+							if _, err := os.Stat(c.String("file")); os.IsNotExist(err) {
+								fmt.Printf("The specified stream defintion file %s does not existed.", sfile)
+								return nil
+							}
+							fmt.Printf("Creating a new stream from file %s", sfile)
+							if stream, err := ioutil.ReadFile(sfile); err != nil {
+								fmt.Printf("Failed to read from stream definition file %s", sfile)
+								return nil
+							} else {
+								args := strings.Join([]string{"CREATE STREAM ", string(stream)}, " ")
+								return streamProcess(client, args)
+							}
+							return nil
+						} else {
+							return streamProcess(client, "")
+						}
+					},
+				},
+				{
+					Name:  "rule",
+					Usage: "create rule $rule_name [$rule_json | -f rule_def_file]",
+					Flags: []cli.Flag {
+						cli.StringFlag{
+							Name: "file, f",
+							Usage: "the location of rule definition file",
+							FilePath: "/home/myrule.txt",
+						},
+					},
+					Action: func(c *cli.Context) error {
+						sfile := c.String("file")
+						if sfile != "" {
+							if _, err := os.Stat(c.String("file")); os.IsNotExist(err) {
+								fmt.Printf("The specified rule defenition file %s does not existed.", sfile)
+								return nil
+							}
+							fmt.Printf("Creating a new rule from file %s", sfile)
+							if rule, err := ioutil.ReadFile(sfile); err != nil {
+								fmt.Printf("Failed to read from rule definition file %s", sfile)
+								return nil
+							} else {
+								if len(c.Args()) != 1 {
+									fmt.Printf("Expect rule name.\n")
+									return nil
+								}
+								rname := c.Args()[0]
+								var reply string
+								args := &common.Rule{rname, string(rule)}
+								err = client.Call("Server.CreateRule", args, &reply)
+								if err != nil {
+									fmt.Println(err)
+								} else {
+									fmt.Println(reply)
+								}
+							}
+							return nil
+						} else {
+							if len(c.Args()) != 2 {
+								fmt.Printf("Expect rule name and json.\nBut found %d args:%s.\n", len(c.Args()), c.Args())
+								return nil
+							}
+							rname := c.Args()[0]
+							rjson := c.Args()[1]
+							var reply string
+							args := &common.Rule{rname, rjson}
+							err = client.Call("Server.CreateRule", args, &reply)
+							if err != nil {
+								fmt.Println(err)
+							} else {
+								fmt.Println(reply)
+							}
+							return nil
+						}
+					},
+				},
+			},
+
+		},
 		{
-			Name:      "query",
-			Aliases:   []string{"q"},
-			Usage:     "query against stream",
-			Action:    func(c *cli.Context) error {
-				reader := bufio.NewReader(os.Stdin)
-				var inputs []string
-				for {
-					fmt.Print("xstream > ")
+			Name:      "describe",
+			Aliases:   []string{"describe"},
+			Usage:     "describe stream $stream_name | describe rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "stream",
+					Usage: "describe stream $stream_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						return streamProcess(client, "")
+					},
+				},
+				{
+					Name:  "rule",
+					Usage: "describe rule $rule_name",
+					Action:    func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.DescRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
+			},
+		},
 
-					text, _ := reader.ReadString('\n')
-					inputs = append(inputs, text)
-					// convert CRLF to LF
-					text = strings.Replace(text, "\n", "", -1)
+		{
+			Name:        "drop",
+			Aliases:     []string{"drop"},
+			Usage:       "drop stream $stream_name | drop rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "stream",
+					Usage: "drop stream $stream_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						return streamProcess(client, "")
+					},
+				},
+				{
+					Name:  "rule",
+					Usage: "drop rule $rule_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.DropRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
+			},
+		},
 
-					if strings.ToLower(text) == "quit" || strings.ToLower(text) == "exit" {
-						break
-					} else {
-						fmt.Println(text)
+		{
+			Name:      "show",
+			Aliases:   []string{"show"},
+			Usage:     "show streams | show rules",
 
-						err = processors.NewRuleProcessor(text, dataDir).Exec()
+			Subcommands: []cli.Command{
+				{
+					Name:  "streams",
+					Usage: "show streams",
+					Action: func(c *cli.Context) error {
+						return streamProcess(client, "")
+					},
+				},
+				{
+					Name:  "rules",
+					Usage: "show rules",
+					Action:    func(c *cli.Context) error {
+						var reply string
+						err = client.Call("Server.ShowRules", 0, &reply)
 						if err != nil {
-							fmt.Printf("create topology error : %s\n", err)
-						}else{
-							fmt.Println("topology running")
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
 						}
-					}
-				}
-				return nil
+						return nil
+					},
+				},
+			},
+		},
+
+		{
+			Name:        "getstatus",
+			Aliases:     []string{"getstatus"},
+			Usage:       "getstatus rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "rule",
+					Usage: "getstatus rule $rule_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.GetStatusRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
+			},
+		},
+		{
+			Name:        "start",
+			Aliases:     []string{"start"},
+			Usage:       "start rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "rule",
+					Usage: "start rule $rule_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.StartRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
+			},
+		},
+		{
+			Name:        "stop",
+			Aliases:     []string{"stop"},
+			Usage:       "stop rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "rule",
+					Usage: "stop rule $rule_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.StopRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
+			},
+		},
+		{
+			Name:        "restart",
+			Aliases:     []string{"restart"},
+			Usage:       "restart rule $rule_name",
+			Subcommands: []cli.Command{
+				{
+					Name:  "rule",
+					Usage: "restart rule $rule_name",
+					//Flags: nflag,
+					Action: func(c *cli.Context) error {
+						if len(c.Args()) != 1 {
+							fmt.Printf("Expect rule name.\n")
+							return nil
+						}
+						rname := c.Args()[0]
+						var reply string
+						err = client.Call("Server.RestartRule", rname, &reply)
+						if err != nil {
+							fmt.Println(err)
+						} else {
+							fmt.Println(reply)
+						}
+						return nil
+					},
+				},
 			},
 		},
 	}
@@ -110,6 +442,6 @@ func main() {
 
 	err = app.Run(os.Args)
 	if err != nil {
-		log.Fatal(err)
+		fmt.Errorf("%s", err)
 	}
 }

xstream/collectors/func.go

@@ -6,11 +6,10 @@ import (
 	"errors"
 )
 
-var log = common.Log
 // CollectorFunc is a function used to colllect
 // incoming stream data. It can be used as a
 // stream sink.
-type CollectorFunc func(interface{}) error
+type CollectorFunc func(context.Context, interface{}) error
 
 // FuncCollector is a colletor that uses a function
 // to collect data.  The specified function must be
@@ -27,8 +26,8 @@ type FuncCollector struct {
 // Func creates a new value *FuncCollector that
 // will use the specified function parameter to
 // collect streaming data.
-func Func(f CollectorFunc) *FuncCollector {
-	return &FuncCollector{f: f, input: make(chan interface{}, 1024)}
+func Func(name string, f CollectorFunc) *FuncCollector {
+	return &FuncCollector{f: f, name:name, input: make(chan interface{}, 1024)}
 }
 
 func (c *FuncCollector) GetName() string  {
@@ -40,40 +39,29 @@ func (c *FuncCollector) GetInput() (chan<- interface{}, string)  {
 }
 
 // Open is the starting point that starts the collector
-func (c *FuncCollector) Open(ctx context.Context) <-chan error {
+func (c *FuncCollector) Open(ctx context.Context, result chan<- error) {
 	//c.logf = autoctx.GetLogFunc(ctx)
 	//c.errf = autoctx.GetErrFunc(ctx)
-
+	log := common.GetLogger(ctx)
 	log.Println("Opening func collector")
-	result := make(chan error)
 
 	if c.f == nil {
 		err := errors.New("Func collector missing function")
 		log.Println(err)
 		go func() { result <- err }()
-		return result
 	}
 
 	go func() {
-		defer func() {
-			log.Println("Closing func collector")
-			close(result)
-		}()
-
 		for {
 			select {
-			case item, opened := <-c.input:
-				if !opened {
-					return
-				}
-				if err := c.f(item); err != nil {
+			case item := <-c.input:
+				if err := c.f(ctx, item); err != nil {
 					log.Println(err)
 				}
 			case <-ctx.Done():
+				log.Infof("Func collector %s done", c.name)
 				return
 			}
 		}
 	}()
-
-	return result
 }

xstream/demo/func_visitor.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"engine/xsql"
+	"fmt"
+	"strings"
+
+)
+
+func main() {
+	stmt, _ := xsql.NewParser(strings.NewReader("SELECT id1 FROM src1 left join src2 on src1.f1->cid = src2.f2->cid")).Parse()
+
+	var srcs []string
+	xsql.WalkFunc(stmt.Joins, func(node xsql.Node) {
+		if f,ok := node.(*xsql.FieldRef); ok {
+			if string(f.StreamName) == "" {
+				return
+			}
+			srcs = append(srcs, string(f.StreamName))
+		}
+	});
+
+	for _, src := range srcs {
+		fmt.Println(src)
+	}
+}

xstream/extensions/mqtt_source.go

@@ -2,19 +2,17 @@ package extensions
 
 import (
 	"context"
+	"encoding/json"
 	"engine/common"
 	"engine/xsql"
 	"fmt"
 	MQTT "github.com/eclipse/paho.mqtt.golang"
 	"github.com/go-yaml/yaml"
 	"github.com/google/uuid"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"time"
 )
 
-var log = common.Log
 type MQTTSource struct {
 	srv      string
 	tpc      string
@@ -39,26 +37,10 @@ type MQTTConfig struct {
 const confName string = "mqtt_source.yaml"
 
 func NewWithName(name string, topic string, confKey string) (*MQTTSource, error) {
-	confDir, err := common.GetConfLoc()
-	var file string = confDir + confName
-	if err != nil {
-		//Try the development mode, read from workspace
-		file = "xstream/extensions/" + confName
-		if abs, err1 := filepath.Abs(file); err1 != nil {
-			return nil, err1
-		} else {
-			file = abs
-		}
-	}
-
-	b, err := ioutil.ReadFile(file)
-	if err != nil {
-		log.Fatal(err)
-	}
-
+	b := common.LoadConf(confName)
 	var cfg map[string]MQTTConfig
 	if err := yaml.Unmarshal(b, &cfg); err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 
 	ms := &MQTTSource{tpc: topic, name: name}
@@ -104,18 +86,19 @@ func (ms *MQTTSource) AddOutput(output chan<- interface{}, name string) {
 	if _, ok := ms.outs[name]; !ok{
 		ms.outs[name] = output
 	}else{
-		log.Error("fail to add output %s, operator %s already has an output of the same name", name, ms.name)
+		common.Log.Warnf("fail to add output %s, operator %s already has an output of the same name", name, ms.name)
 	}
 }
 
 func (ms *MQTTSource) Open(ctx context.Context) error {
+	log := common.GetLogger(ctx)
 	go func() {
 		exeCtx, cancel := context.WithCancel(ctx)
 		opts := MQTT.NewClientOptions().AddBroker(ms.srv)
 
 		if ms.clientid == "" {
 			if uuid, err := uuid.NewUUID(); err != nil {
-				log.Printf("Failed to get uuid, the error is %s.\n", err)
+				log.Printf("Failed to get uuid, the error is %s", err)
 				cancel()
 				return
 			} else {
@@ -127,15 +110,19 @@ func (ms *MQTTSource) Open(ctx context.Context) error {
 
 		h := func(client MQTT.Client, msg MQTT.Message) {
 			if ms.tpc != msg.Topic() {
-				select {
-				case <-exeCtx.Done():
-					log.Println("Done 1.")
-					ms.conn.Disconnect(5000)
-				}
 				return
 			} else {
 				log.Infof("received %s", msg.Payload())
-				tuple := &xsql.Tuple{EmitterName:ms.name, Message:msg.Payload(), Timestamp: common.TimeToUnixMilli(time.Now())}
+
+				result := make(map[string]interface{})
+				//The unmarshal type can only be bool, float64, string, []interface{}, map[string]interface{}, nil
+				if e := json.Unmarshal(msg.Payload(), &result); e != nil {
+					log.Errorf("Invalid data format, cannot convert %s into JSON with error %s", string(msg.Payload()), e)
+					return
+				}
+				//Convert the keys to lowercase
+				result = xsql.LowercaseKeyMap(result)
+				tuple := &xsql.Tuple{Emitter: ms.tpc, Message:result, Timestamp: common.TimeToUnixMilli(time.Now())}
 				for _, out := range ms.outs{
 					out <- tuple
 				}
@@ -145,16 +132,24 @@ func (ms *MQTTSource) Open(ctx context.Context) error {
 		opts.SetDefaultPublishHandler(h)
 		c := MQTT.NewClient(opts)
 		if token := c.Connect(); token.Wait() && token.Error() != nil {
-			log.Fatalf("Found error: %s.\n", token.Error())
+			log.Printf("Found error when connecting to %s for %s: %s", ms.srv, ms.name, token.Error())
 			cancel()
+			return
 		}
-		log.Printf("The connection to server %s was established successfully.\n", ms.srv)
+		log.Printf("The connection to server %s was established successfully", ms.srv)
 		ms.conn = c
 		if token := c.Subscribe(ms.tpc, 0, nil); token.Wait() && token.Error() != nil {
-			log.Fatalf("Found error: %s.\n", token.Error())
+			log.Printf("Found error: %s", token.Error())
+			cancel()
+			return
+		}
+		log.Printf("Successfully subscribe to topic %s", ms.tpc)
+		select {
+		case <-exeCtx.Done():
+			log.Println("Mqtt Source Done")
+			ms.conn.Disconnect(5000)
 			cancel()
 		}
-		log.Printf("Successfully subscribe to topic %s.\n", ms.tpc)
 	}()
 
 	return nil

xstream/operators/operations.go

@@ -7,8 +7,6 @@ import (
 	"sync"
 )
 
-var log = common.Log
-
 // UnOperation interface represents unary operations (i.e. Map, Filter, etc)
 type UnOperation interface {
 	Apply(ctx context.Context, data interface{}) interface{}
@@ -67,7 +65,7 @@ func (o *UnaryOperator) AddOutput(output chan<- interface{}, name string) {
 	if _, ok := o.outputs[name]; !ok{
 		o.outputs[name] = output
 	}else{
-		log.Error("fail to add output %s, operator %s already has an output of the same name", name, o.name)
+		common.Log.Warnf("fail to add output %s, operator %s already has an output of the same name", name, o.name)
 	}
 }
 
@@ -77,11 +75,11 @@ func (o *UnaryOperator) GetInput() (chan<- interface{}, string) {
 
 // Exec is the entry point for the executor
 func (o *UnaryOperator) Exec(ctx context.Context) (err error) {
-
-	log.Printf("Unary operator %s is started.\n", o.name)
+	log := common.GetLogger(ctx)
+	log.Printf("Unary operator %s is started", o.name)
 
 	if len(o.outputs) <= 0 {
-		err = fmt.Errorf("No output channel found")
+		err = fmt.Errorf("no output channel found")
 		return
 	}
 
@@ -115,7 +113,7 @@ func (o *UnaryOperator) Exec(ctx context.Context) (err error) {
 				return
 			}
 		case <-ctx.Done():
-			log.Print("UnaryOp %s done.", o.name)
+			log.Printf("UnaryOp %s done.", o.name)
 			return
 		}
 	}()
@@ -124,6 +122,7 @@ func (o *UnaryOperator) Exec(ctx context.Context) (err error) {
 }
 
 func (o *UnaryOperator) doOp(ctx context.Context) {
+	log := common.GetLogger(ctx)
 	if o.op == nil {
 		log.Println("Unary operator missing operation")
 		return
@@ -131,18 +130,14 @@ func (o *UnaryOperator) doOp(ctx context.Context) {
 	exeCtx, cancel := context.WithCancel(ctx)
 
 	defer func() {
-		log.Println("unary operator done, cancelling future items")
+		log.Infof("unary operator %s done, cancelling future items", o.name)
 		cancel()
 	}()
 
 	for {
 		select {
 		// process incoming item
-		case item, opened := <-o.input:
-			if !opened {
-				return
-			}
-
+		case item := <-o.input:
 			result := o.op.Apply(exeCtx, item)
 
 			switch val := result.(type) {
@@ -174,13 +169,15 @@ func (o *UnaryOperator) doOp(ctx context.Context) {
 
 			default:
 				for _, output := range o.outputs{
-					output <- val
+					select {
+					case output <- val:
+					}
 				}
 			}
 
 		// is cancelling
 		case <-exeCtx.Done():
-			log.Println("Cancelling....")
+			log.Printf("unary operator %s cancelling....", o.name)
 			o.mutex.Lock()
 			cancel()
 			o.cancelled = true

xstream/operators/watermark.go

@@ -0,0 +1,258 @@
+package operators
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+	"fmt"
+	"math"
+	"sort"
+	"time"
+)
+
+type WatermarkTuple struct{
+	Timestamp int64
+}
+
+func (t *WatermarkTuple) GetTimestamp() int64 {
+	return t.Timestamp
+}
+
+func (t *WatermarkTuple) IsWatermark() bool {
+	return true
+}
+
+type WatermarkGenerator struct {
+	lastWatermarkTs int64
+	inputTopics []string
+	topicToTs map[string]int64
+	window      *WindowConfig
+	lateTolerance int64
+	interval	int
+	ticker 		common.Ticker
+	stream 	chan<- interface{}
+}
+
+func NewWatermarkGenerator(window *WindowConfig, l int64, s []string, stream chan<- interface{}) (*WatermarkGenerator, error){
+	w := &WatermarkGenerator{
+		window: window,
+		topicToTs: make(map[string]int64),
+		lateTolerance: l,
+		inputTopics: s,
+		stream: stream,
+	}
+	//Tickers to update watermark
+	switch window.Type{
+	case xsql.NOT_WINDOW:
+	case xsql.TUMBLING_WINDOW:
+		w.ticker = common.GetTicker(window.Length)
+		w.interval = window.Length
+	case xsql.HOPPING_WINDOW:
+		w.ticker = common.GetTicker(window.Interval)
+		w.interval = window.Interval
+	case xsql.SLIDING_WINDOW:
+		w.interval = window.Length
+	case xsql.SESSION_WINDOW:
+		//Use timeout to update watermark
+		w.ticker = common.GetTicker(window.Interval)
+		w.interval = window.Interval
+	default:
+		return nil, fmt.Errorf("unsupported window type %d", window.Type)
+	}
+	return w, nil
+}
+
+func (w *WatermarkGenerator) track(s string, ts int64, ctx context.Context) bool {
+	log := common.GetLogger(ctx)
+	log.Infof("watermark generator track event from topic %s at %d", s, ts)
+	currentVal, ok := w.topicToTs[s]
+	if !ok || ts > currentVal {
+		w.topicToTs[s] = ts
+	}
+	r := ts >= w.lastWatermarkTs
+	if r{
+		switch w.window.Type{
+		case xsql.SLIDING_WINDOW:
+			w.trigger(ctx)
+		}
+	}
+	return r
+}
+
+func (w *WatermarkGenerator) start(ctx context.Context) {
+	exeCtx, cancel := context.WithCancel(ctx)
+	log := common.GetLogger(ctx)
+	var c <-chan time.Time
+
+	if w.ticker != nil {
+		c = w.ticker.GetC()
+	}
+	for {
+		select {
+		case <-c:
+			w.trigger(ctx)
+		case <-exeCtx.Done():
+			log.Println("Cancelling watermark generator....")
+			if w.ticker != nil{
+				w.ticker.Stop()
+			}
+			cancel()
+			return
+		}
+	}
+}
+
+func (w *WatermarkGenerator) trigger(ctx context.Context) {
+	log := common.GetLogger(ctx)
+	watermark := w.computeWatermarkTs(ctx)
+	log.Infof("compute watermark event at %d with last %d", watermark, w.lastWatermarkTs)
+	if watermark > w.lastWatermarkTs {
+		t := &WatermarkTuple{Timestamp: watermark}
+		select {
+		case w.stream <- t:
+		default: //TODO need to set buffer
+		}
+		w.lastWatermarkTs = watermark
+		log.Infof("scan watermark event at %d", watermark)
+	}
+}
+
+func (w *WatermarkGenerator) computeWatermarkTs(ctx context.Context) int64{
+	var ts int64
+	if len(w.topicToTs) >= len(w.inputTopics) {
+		ts = math.MaxInt64
+		for _, key := range w.inputTopics {
+			if ts > w.topicToTs[key]{
+				ts = w.topicToTs[key]
+			}
+		}
+	}
+	return ts - w.lateTolerance
+}
+
+//If window end cannot be determined yet, return max int64 so that it can be recalculated for the next watermark
+func (w *WatermarkGenerator) getNextWindow(inputs []*xsql.Tuple,current int64, watermark int64, triggered bool) int64{
+	switch w.window.Type{
+	case xsql.TUMBLING_WINDOW, xsql.HOPPING_WINDOW:
+		if triggered{
+			return current + int64(w.interval)
+		}else{
+			interval := int64(w.interval)
+			nextTs := getEarliestEventTs(inputs, current, watermark)
+			if nextTs == math.MaxInt64 ||  nextTs % interval == 0{
+				return nextTs
+			}
+			return nextTs + (interval - nextTs % interval)
+		}
+	case xsql.SLIDING_WINDOW:
+		nextTs := getEarliestEventTs(inputs, current, watermark)
+		return nextTs
+	case xsql.SESSION_WINDOW:
+		if len(inputs) > 0{
+			timeout, duration := int64(w.window.Interval), int64(w.window.Length)
+			sort.SliceStable(inputs, func(i, j int) bool {
+				return inputs[i].Timestamp < inputs[j].Timestamp
+			})
+			et := inputs[0].Timestamp
+			tick := et + (duration - et % duration)
+			if et % duration == 0 {
+				tick = et
+			}
+			var p int64
+			for _, tuple := range inputs {
+				var r int64 = math.MaxInt64
+				if p > 0{
+					if tuple.Timestamp - p > timeout{
+						r = p + timeout
+					}
+				}
+				if tuple.Timestamp > tick {
+					if tick - duration > et && tick < r {
+						r = tick
+					}
+					tick += duration
+				}
+				if r < math.MaxInt64{
+					return r
+				}
+				p = tuple.Timestamp
+			}
+		}
+		return math.MaxInt64
+	default:
+		return math.MaxInt64
+	}
+}
+
+func (o *WindowOperator) execEventWindow(ctx context.Context) {
+	exeCtx, cancel := context.WithCancel(ctx)
+	log := common.GetLogger(ctx)
+	go o.watermarkGenerator.start(ctx)
+	var (
+		inputs []*xsql.Tuple
+		triggered bool
+		nextWindowEndTs int64
+		prevWindowEndTs int64
+	)
+
+	for {
+		select {
+		// process incoming item
+		case item, opened := <-o.input:
+			if !opened {
+				break
+			}
+			if d, ok := item.(xsql.Event); !ok {
+				log.Errorf("Expect xsql.Event type")
+				break
+			}else{
+				if d.IsWatermark(){
+					watermarkTs := d.GetTimestamp()
+					windowEndTs := nextWindowEndTs
+					//Session window needs a recalculation of window because its window end depends the inputs
+					if windowEndTs == math.MaxInt64 || o.window.Type == xsql.SESSION_WINDOW || o.window.Type == xsql.SLIDING_WINDOW{
+						windowEndTs = o.watermarkGenerator.getNextWindow(inputs, prevWindowEndTs, watermarkTs, triggered)
+					}
+					for windowEndTs <= watermarkTs && windowEndTs >= 0 {
+						log.Debugf("Window end ts %d Watermark ts %d", windowEndTs, watermarkTs)
+						log.Debugf("Current input count %d", len(inputs))
+						//scan all events and find out the event in the current window
+						inputs, triggered = o.scan(inputs, windowEndTs, ctx)
+						prevWindowEndTs = windowEndTs
+						windowEndTs = o.watermarkGenerator.getNextWindow(inputs, windowEndTs, watermarkTs, triggered)
+					}
+					nextWindowEndTs = windowEndTs
+					log.Debugf("next window end %d", nextWindowEndTs)
+				}else{
+					tuple, ok := d.(*xsql.Tuple)
+					if !ok{
+						log.Infof("receive non tuple element %v", d)
+					}
+					log.Debugf("event window receive tuple %s", tuple.Message)
+					if o.watermarkGenerator.track(tuple.Emitter, d.GetTimestamp(),ctx){
+						inputs = append(inputs, tuple)
+					}
+				}
+
+			}
+		// is cancelling
+		case <-exeCtx.Done():
+			log.Println("Cancelling window....")
+			if o.ticker != nil{
+				o.ticker.Stop()
+			}
+			cancel()
+			return
+		}
+	}
+}
+
+func getEarliestEventTs(inputs []*xsql.Tuple, startTs int64, endTs int64) int64{
+	var minTs int64 = math.MaxInt64
+	for _, t := range inputs{
+		if t.Timestamp > startTs && t.Timestamp <= endTs && t.Timestamp < minTs{
+			minTs = t.Timestamp
+		}
+	}
+	return minTs
+}

xstream/operators/window_op.go

@@ -5,59 +5,74 @@ import (
 	"engine/common"
 	"engine/xsql"
 	"fmt"
+	"github.com/sirupsen/logrus"
 	"math"
 	"time"
 )
 
-type WindowType int
-const (
-	NO_WINDOW WindowType = iota
-	TUMBLING_WINDOW
-	HOPPING_WINDOW
-	SLIDING_WINDOW
-	SESSION_WINDOW
-)
-
 type WindowConfig struct {
-	Type WindowType
-	Length int64
-	Interval int64   //If interval is not set, it is equals to Length
+	Type xsql.WindowType
+	Length int
+	Interval int   //If interval is not set, it is equals to Length
 }
 
 type WindowOperator struct {
 	input       chan interface{}
 	outputs     map[string]chan<- interface{}
 	name 		string
-	ticker 		*time.Ticker
+	ticker 		common.Ticker  //For processing time only
 	window      *WindowConfig
-	interval	int64
+	interval	int
 	triggerTime int64
+	isEventTime bool
+	watermarkGenerator *WatermarkGenerator //For event time only
 }
 
-func NewWindowOp(name string, config *WindowConfig) *WindowOperator {
+func NewWindowOp(name string, w *xsql.Window, isEventTime bool, lateTolerance int64, streams []string) (*WindowOperator, error) {
 	o := new(WindowOperator)
 
 	o.input = make(chan interface{}, 1024)
 	o.outputs = make(map[string]chan<- interface{})
 	o.name = name
-	o.window = config
-	switch config.Type{
-	case NO_WINDOW:
-	case TUMBLING_WINDOW:
-		o.ticker = time.NewTicker(time.Duration(config.Length) * time.Millisecond)
-		o.interval = config.Length
-	case HOPPING_WINDOW:
-		o.ticker = time.NewTicker(time.Duration(config.Interval) * time.Millisecond)
-		o.interval = config.Interval
-	case SLIDING_WINDOW:
-		o.interval = config.Length
-	case SESSION_WINDOW:
-		o.interval = config.Interval
-	default:
-		log.Errorf("Unsupported window type %d", config.Type)
+	o.isEventTime = isEventTime
+	if w != nil{
+		o.window = &WindowConfig{
+			Type: w.WindowType,
+			Length: w.Length.Val,
+			Interval: w.Interval.Val,
+		}
+	}else{
+		o.window = &WindowConfig{
+			Type: xsql.NOT_WINDOW,
+		}
 	}
 
-	return o
+	if isEventTime{
+		//Create watermark generator
+		if w, err := NewWatermarkGenerator(o.window, lateTolerance, streams, o.input); err != nil{
+			return nil, err
+		}else{
+			o.watermarkGenerator = w
+		}
+	}else{
+		switch o.window.Type{
+		case xsql.NOT_WINDOW:
+		case xsql.TUMBLING_WINDOW:
+			o.ticker = common.GetTicker(o.window.Length)
+			o.interval = o.window.Length
+		case xsql.HOPPING_WINDOW:
+			o.ticker = common.GetTicker(o.window.Interval)
+			o.interval = o.window.Interval
+		case xsql.SLIDING_WINDOW:
+			o.interval = o.window.Length
+		case xsql.SESSION_WINDOW:
+			o.ticker = common.GetTicker(o.window.Length)
+			o.interval = o.window.Interval
+		default:
+			return nil, fmt.Errorf("unsupported window type %d", o.window.Type)
+		}
+	}
+	return o, nil
 }
 
 func (o *WindowOperator) GetName() string {
@@ -68,7 +83,7 @@ func (o *WindowOperator) AddOutput(output chan<- interface{}, name string) {
 	if _, ok := o.outputs[name]; !ok{
 		o.outputs[name] = output
 	}else{
-		log.Error("fail to add output %s, operator %s already has an output of the same name", name, o.name)
+		common.Log.Warnf("fail to add output %s, operator %s already has an output of the same name", name, o.name)
 	}
 }
 
@@ -77,121 +92,165 @@ func (o *WindowOperator) GetInput() (chan<- interface{}, string) {
 }
 
 // Exec is the entry point for the executor
+// input: *xsql.Tuple from preprocessor
+// output: xsql.WindowTuplesSet
 func (o *WindowOperator) Exec(ctx context.Context) (err error) {
-
-	log.Printf("Window operator %s is started.\n", o.name)
+	log := common.GetLogger(ctx)
+	log.Printf("Window operator %s is started", o.name)
 
 	if len(o.outputs) <= 0 {
 		err = fmt.Errorf("no output channel found")
 		return
 	}
+	if o.isEventTime{
+		go o.execEventWindow(ctx)
+	}else{
+		go o.execProcessingWindow(ctx)
+	}
 
-	go func() {
-		var (
-			inputs []*xsql.Tuple
-			c <-chan time.Time
-			timeoutTicker *time.Timer
-			timeout <-chan time.Time
-		)
+	return nil
+}
 
-		if o.ticker != nil {
-			c = o.ticker.C
-		}
+func (o *WindowOperator) execProcessingWindow(ctx context.Context) {
+	exeCtx, cancel := context.WithCancel(ctx)
+	log := common.GetLogger(ctx)
+	var (
+		inputs []*xsql.Tuple
+		c <-chan time.Time
+		timeoutTicker common.Timer
+		timeout <-chan time.Time
+	)
 
-		for {
-			select {
-			// process incoming item
-			case item, opened := <-o.input:
-				if !opened {
-					return
-				}
-				if d, ok := item.(*xsql.Tuple); !ok {
-					log.Errorf("Expect xsql.Tuple type.\n")
-					return
-				}else{
-					inputs = append(inputs, d)
-					switch o.window.Type{
-					case NO_WINDOW:
-						inputs = o.trigger(inputs, d.Timestamp)
-					case SLIDING_WINDOW:
-						inputs = o.trigger(inputs, d.Timestamp)
-					case SESSION_WINDOW:
-						if o.ticker == nil{ //Stopped by timeout or init
-							o.ticker = time.NewTicker(time.Duration(o.window.Length) * time.Millisecond)
-							c = o.ticker.C
-						}
-						if timeoutTicker != nil {
-							timeoutTicker.Stop()
-							timeoutTicker.Reset(time.Duration(o.window.Interval) * time.Millisecond)
-						} else {
-							timeoutTicker = time.NewTimer(time.Duration(o.window.Interval) * time.Millisecond)
-							timeout = timeoutTicker.C
-						}
+	if o.ticker != nil {
+		c = o.ticker.GetC()
+	}
+
+	for {
+		select {
+		// process incoming item
+		case item, opened := <-o.input:
+			if !opened {
+				break
+			}
+			if d, ok := item.(*xsql.Tuple); !ok {
+				log.Errorf("Expect xsql.Tuple type")
+				break
+			}else{
+				log.Debugf("Event window receive tuple %s", d.Message)
+				inputs = append(inputs, d)
+				switch o.window.Type{
+				case xsql.NOT_WINDOW:
+					inputs, _ = o.scan(inputs, d.Timestamp, ctx)
+				case xsql.SLIDING_WINDOW:
+					inputs, _ = o.scan(inputs, d.Timestamp, ctx)
+				case xsql.SESSION_WINDOW:
+					if timeoutTicker != nil {
+						timeoutTicker.Stop()
+						timeoutTicker.Reset(time.Duration(o.window.Interval) * time.Millisecond)
+					} else {
+						timeoutTicker = common.GetTimer(o.window.Interval)
+						timeout = timeoutTicker.GetC()
 					}
 				}
-			case now := <-c:
-				if len(inputs) > 0 {
-					log.Infof("triggered by ticker")
-					inputs = o.trigger(inputs, common.TimeToUnixMilli(now))
-				}
-			case now := <-timeout:
-				if len(inputs) > 0 {
-					log.Infof("triggered by timeout")
-					inputs = o.trigger(inputs, common.TimeToUnixMilli(now))
+			}
+		case now := <-c:
+			if len(inputs) > 0 {
+				n := common.TimeToUnixMilli(now)
+				//For session window, check if the last scan time is newer than the inputs
+				if o.window.Type == xsql.SESSION_WINDOW{
+					//scan time for session window will record all triggers of the ticker but not the timeout
+					lastTriggerTime := o.triggerTime
+					o.triggerTime = n
+					//Check if the current window has exceeded the max duration, if not continue expand
+					if lastTriggerTime < inputs[0].Timestamp{
+						break
+					}
 				}
+				log.Infof("triggered by ticker")
+				inputs, _ = o.scan(inputs, n, ctx)
+			}
+		case now := <-timeout:
+			if len(inputs) > 0 {
+				log.Infof("triggered by timeout")
+				inputs, _ = o.scan(inputs, common.TimeToUnixMilli(now), ctx)
+				//expire all inputs, so that when timer scan there is no item
+				inputs = make([]*xsql.Tuple, 0)
+			}
+		// is cancelling
+		case <-exeCtx.Done():
+			log.Println("Cancelling window....")
+			if o.ticker != nil{
 				o.ticker.Stop()
-				o.ticker = nil
-			// is cancelling
-			case <-ctx.Done():
-				log.Println("Cancelling....")
-				o.ticker.Stop()
-				return
 			}
+			cancel()
+			return
 		}
-	}()
-
-	return nil
+	}
 }
 
-func (o *WindowOperator) trigger(inputs []*xsql.Tuple, triggerTime int64) []*xsql.Tuple{
-	log.Printf("window %s triggered at %s", o.name, triggerTime)
+func (o *WindowOperator) scan(inputs []*xsql.Tuple, triggerTime int64, ctx context.Context) ([]*xsql.Tuple, bool){
+	log := common.GetLogger(ctx)
+	log.Printf("window %s triggered at %s", o.name, time.Unix(triggerTime/1000, triggerTime%1000))
 	var delta int64
-	if o.window.Type == HOPPING_WINDOW || o.window.Type == SLIDING_WINDOW {
-		lastTriggerTime := o.triggerTime
-		o.triggerTime = triggerTime
-		if lastTriggerTime <= 0 {
-			delta = math.MaxInt32  //max int, all events for the initial window
-		}else{
-			delta = o.triggerTime - lastTriggerTime - o.window.Interval
-			if delta > 100 && o.window.Interval > 0 {
-				log.Warnf("Possible long computation in window; Previous eviction time: %d, current eviction time: %d", lastTriggerTime, o.triggerTime)
-			}
-		}
+	if o.window.Type == xsql.HOPPING_WINDOW || o.window.Type == xsql.SLIDING_WINDOW {
+		delta = o.calDelta(triggerTime, delta, log)
 	}
-	var results xsql.MultiEmitterTuples = make([]xsql.EmitterTuples, 0)
+	var results xsql.WindowTuplesSet = make([]xsql.WindowTuples, 0)
 	i := 0
 	//Sync table
-	for _, tuple := range inputs{
-		if o.window.Type == HOPPING_WINDOW || o.window.Type == SLIDING_WINDOW {
+	for _, tuple := range inputs {
+		if o.window.Type == xsql.HOPPING_WINDOW || o.window.Type == xsql.SLIDING_WINDOW {
 			diff := o.triggerTime - tuple.Timestamp
-			if diff >= o.window.Length + delta {
+			if diff > int64(o.window.Length)+delta {
 				log.Infof("diff: %d, length: %d, delta: %d", diff, o.window.Length, delta)
 				log.Infof("tuple %s emitted at %d expired", tuple, tuple.Timestamp)
 				//Expired tuple, remove it by not adding back to inputs
 				continue
 			}
-			//All tuples in tumbling window are not added back
+			//Added back all inputs for non expired events
 			inputs[i] = tuple
 			i++
+		} else if tuple.Timestamp > triggerTime {
+			//Only added back early arrived events
+			inputs[i] = tuple
+			i++
+		}
+		if tuple.Timestamp <= triggerTime{
+			results = results.AddTuple(tuple)
 		}
-		results.AddTuple(tuple)
 	}
-	if len(results) > 0{
-		log.Printf("window %s triggered for %d tuples", o.name, len(results))
-		for _, output := range o.outputs{
-			output <- results
+	triggered := false
+	if len(results) > 0 {
+		log.Printf("window %s triggered for %d tuples", o.name, len(inputs))
+		if o.isEventTime{
+			results.Sort()
+		}
+		for _, output := range o.outputs {
+			select {
+			case output <- results:
+				triggered = true
+			default: //TODO need to set buffer
+			}
 		}
 	}
 
-	return inputs[:i]
+	return inputs[:i], triggered
+}
+
+func (o *WindowOperator) calDelta(triggerTime int64, delta int64, log *logrus.Entry) int64 {
+	lastTriggerTime := o.triggerTime
+	o.triggerTime = triggerTime
+	if lastTriggerTime <= 0 {
+		delta = math.MaxInt16 //max int, all events for the initial window
+	} else {
+		if !o.isEventTime && o.window.Interval > 0 {
+			delta = o.triggerTime - lastTriggerTime - int64(o.window.Interval)
+			if delta > 100 {
+				log.Warnf("Possible long computation in window; Previous eviction time: %d, current eviction time: %d", lastTriggerTime, o.triggerTime)
+			}
+		} else {
+			delta = 0
+		}
+	}
+	return delta
 }

xstream/server/main.go

@@ -0,0 +1,311 @@
+package main
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql/processors"
+	"engine/xstream"
+	"engine/xstream/sinks"
+	"fmt"
+	"net"
+	"net/http"
+	"net/rpc"
+	"path"
+	"strings"
+	"time"
+)
+var dataDir string
+var log = common.Log
+
+type RuleState struct{
+	Name string
+	Topology *xstream.TopologyNew
+	Triggered bool
+}
+type RuleRegistry map[string]*RuleState
+var registry RuleRegistry
+var processor *processors.RuleProcessor
+
+type Server int
+
+var QUERY_RULE_ID string = "internal-xstream_query_rule"
+func (t *Server) CreateQuery(sql string, reply *string) error {
+	if _, ok := registry[QUERY_RULE_ID]; ok {
+		stopQuery()
+	}
+	tp, err := processors.NewRuleProcessor(path.Dir(dataDir)).ExecQuery(QUERY_RULE_ID, sql)
+	if err != nil {
+		msg := fmt.Sprintf("failed to create query: %s.", err)
+		log.Println(msg)
+		return fmt.Errorf(msg)
+	} else {
+		rs := &RuleState{Name: QUERY_RULE_ID, Topology: tp, Triggered: true}
+		registry[QUERY_RULE_ID] = rs
+		msg := fmt.Sprintf("query is submit successfully.")
+		log.Println(msg)
+		*reply = fmt.Sprintf(msg)
+	}
+	return nil
+}
+
+func stopQuery() {
+	if rs, ok := registry[QUERY_RULE_ID]; ok {
+		log.Printf("stop the query.")
+		(*rs.Topology).Cancel()
+		delete(registry, QUERY_RULE_ID)
+	}
+}
+
+/**
+ * qid is not currently used.
+ */
+func (t *Server) GetQueryResult(qid string, reply *string) error {
+	sinks.QR.LastFetch = time.Now()
+	sinks.QR.Mux.Lock()
+	if len(sinks.QR.Results) > 0 {
+		*reply = strings.Join(sinks.QR.Results, "")
+		sinks.QR.Results = make([]string, 10)
+	} else {
+		*reply = ""
+	}
+	sinks.QR.Mux.Unlock()
+	return nil
+}
+
+
+func (t *Server) Stream(stream string, reply *string) error{
+	content, err := processors.NewStreamProcessor(stream, path.Join(path.Dir(dataDir), "stream")).Exec()
+	if err != nil {
+		fmt.Printf("stream command error: %s\n", err)
+		return err
+	} else {
+		for _, c := range content{
+			*reply = *reply + fmt.Sprintln(c)
+		}
+	}
+	return nil
+}
+
+func (t *Server) CreateRule(rule *common.Rule, reply *string) error{
+	r, err := processor.ExecCreate(rule.Name, rule.Json)
+	if err != nil {
+		return fmt.Errorf("create rule error : %s\n", err)
+	} else {
+		*reply = fmt.Sprintf("rule %s created", rule.Name)
+	}
+	//Start the rule
+	rs, err := t.createRuleState(r)
+	if err != nil {
+		return err
+	}
+	err = t.doStartRule(rs)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *Server) createRuleState(rule *xstream.Rule) (*RuleState, error){
+	if tp, err := processor.ExecInitRule(rule); err != nil{
+		return nil, err
+	}else{
+		rs := &RuleState{
+			Name: rule.Id,
+			Topology: tp,
+			Triggered: true,
+		}
+		registry[rule.Id] = rs
+		return rs, nil
+	}
+}
+
+func (t *Server) GetStatusRule(name string, reply *string) error{
+	if rs, ok := registry[name]; ok{
+		if !rs.Triggered {
+			*reply = "stopped: canceled manually"
+			return nil
+		}
+		c := (*rs.Topology).GetContext()
+		if c != nil{
+			err := c.Err()
+			switch err{
+			case nil:
+				*reply = "running"
+			case context.Canceled:
+				*reply = "stopped: canceled by error"
+			case context.DeadlineExceeded:
+				*reply = "stopped: deadline exceed"
+			default:
+				*reply = "stopped: unknown reason"
+			}
+		}else{
+			*reply = "stopped: no context found"
+		}
+	}else{
+		return fmt.Errorf("rule %s not found", name)
+	}
+	return nil
+}
+
+func (t *Server) StartRule(name string, reply *string) error{
+	var rs *RuleState
+	rs, ok := registry[name]
+	if !ok{
+		r, err := processor.GetRuleByName(name)
+		if err != nil{
+			return err
+		}
+		rs, err = t.createRuleState(r)
+		if err != nil {
+			return err
+		}
+	}
+	err := t.doStartRule(rs)
+	if err != nil{
+		return err
+	}
+	*reply = fmt.Sprintf("rule %s started", name)
+	return nil
+}
+
+func (t *Server) doStartRule(rs *RuleState) error{
+	rs.Triggered = true
+	go func() {
+		tp := rs.Topology
+		select {
+		case err := <-tp.Open():
+			log.Println(err)
+			tp.Cancel()
+		}
+	}()
+	return nil
+}
+
+func (t *Server) StopRule(name string, reply *string) error{
+	if rs, ok := registry[name]; ok{
+		(*rs.Topology).Cancel()
+		rs.Triggered = false
+		*reply = fmt.Sprintf("rule %s stopped", name)
+	}else{
+		*reply = fmt.Sprintf("rule %s not found", name)
+	}
+	return nil
+}
+
+func (t *Server) RestartRule(name string, reply *string) error{
+	err := t.StopRule(name, reply)
+	if err != nil{
+		return err
+	}
+	err = t.StartRule(name, reply)
+	if err != nil{
+		return err
+	}
+	*reply = fmt.Sprintf("rule %s restarted", name)
+	return nil
+}
+
+func (t *Server) DescRule(name string, reply *string) error{
+	r, err := processor.ExecDesc(name)
+	if err != nil {
+		return fmt.Errorf("desc rule error : %s\n", err)
+	} else {
+		*reply = r
+	}
+	return nil
+}
+
+func (t *Server) ShowRules(_ int, reply *string) error{
+	r, err := processor.ExecShow()
+	if err != nil {
+		return fmt.Errorf("show rule error : %s\n", err)
+	} else {
+		*reply = r
+	}
+	return nil
+}
+
+func (t *Server) DropRule(name string, reply *string) error{
+	r, err := processor.ExecDrop(name)
+	if err != nil {
+		return fmt.Errorf("drop rule error : %s\n", err)
+	} else {
+		err := t.StopRule(name, reply)
+		if err != nil{
+			return err
+		}
+	}
+	*reply = r
+	return nil
+}
+
+func init(){
+	var err error
+	dataDir, err = common.GetDataLoc()
+	if err != nil {
+		log.Panic(err)
+	}else{
+		log.Infof("db location is %s", dataDir)
+	}
+
+	processor = processors.NewRuleProcessor(path.Dir(dataDir))
+	registry = make(RuleRegistry)
+
+	ticker := time.NewTicker(time.Second * 5)
+	go func() {
+		for {
+			<-ticker.C
+			if _, ok := registry[QUERY_RULE_ID]; !ok {
+				continue
+			}
+
+			n := time.Now()
+			w := 10 * time.Second
+			if v := n.Sub(sinks.QR.LastFetch); v >= w {
+				log.Printf("The client seems no longer fetch the query result, stop the query now.")
+				stopQuery()
+			}
+		}
+		//defer ticker.Stop()
+	}()
+}
+
+func main() {
+	server := new(Server)
+	//Start rules
+	if rules, err := processor.GetAllRules(); err != nil{
+		log.Infof("Start rules error: %s", err)
+	}else{
+		log.Info("Starting rules")
+		var reply string
+		for _, rule := range rules{
+			err = server.StartRule(rule, &reply)
+			if err != nil {
+				log.Info(err)
+			}else{
+				log.Info(reply)
+			}
+		}
+	}
+
+	//Start server
+	err := rpc.Register(server)
+	if err != nil {
+		log.Fatal("Format of service Server isn't correct. ", err)
+	}
+	// Register a HTTP handler
+	rpc.HandleHTTP()
+	// Listen to TPC connections on port 1234
+	listener, e := net.Listen("tcp", fmt.Sprintf(":%d", common.Config.Port))
+	if e != nil {
+		log.Fatal("Listen error: ", e)
+	}
+	msg := fmt.Sprintf("Serving Rule server on port %d", common.Config.Port)
+	log.Info(msg)
+	fmt.Println(msg)
+	// Start accept incoming HTTP connections
+	err = http.Serve(listener, nil)
+	if err != nil {
+		log.Fatal("Error serving: ", err)
+	}
+}

xstream/sinks/log_sink.go

@@ -0,0 +1,38 @@
+package sinks
+
+import (
+	"context"
+	"engine/common"
+	"engine/xstream/collectors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// log action, no properties now
+// example: {"log":{}}
+func NewLogSink(name string, ruleId string) *collectors.FuncCollector {
+	return collectors.Func(name, func(ctx context.Context, data interface{}) error {
+		log := common.GetLogger(ctx)
+		log.Printf("sink result for rule %s: %s", ruleId, data)
+		return nil
+	})
+}
+
+type QueryResult struct {
+	Results []string
+	LastFetch time.Time
+	Mux sync.Mutex
+}
+
+var QR = &QueryResult{LastFetch:time.Now()}
+
+func NewLogSinkToMemory(name string, ruleId string) *collectors.FuncCollector {
+	QR.Results = make([]string, 10)
+	return collectors.Func(name, func(ctx context.Context, data interface{}) error {
+		QR.Mux.Lock()
+		QR.Results = append(QR.Results, fmt.Sprintf("%s", data))
+		QR.Mux.Unlock()
+		return nil
+	})
+}

xstream/sinks/mqtt_sink.go

@@ -0,0 +1,91 @@
+package sinks
+
+import (
+	"context"
+	"engine/common"
+	"fmt"
+	MQTT "github.com/eclipse/paho.mqtt.golang"
+	"github.com/google/uuid"
+)
+
+type MQTTSink struct {
+	srv      string
+	tpc      string
+	clientid string
+
+	input chan interface{}
+	conn MQTT.Client
+	ruleId   string
+	name 	 string
+	//ctx context.Context
+}
+
+func NewMqttSink(name string, ruleId string, properties interface{}) (*MQTTSink, error) {
+	ps, ok := properties.(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("expect map[string]interface{} type for the mqtt sink properties")
+	}
+	srv, ok := ps["server"]
+	if !ok {
+		return nil, fmt.Errorf("mqtt sink is missing property server")
+	}
+	tpc, ok := ps["topic"]
+	if !ok {
+		return nil, fmt.Errorf("mqtt sink is missing property topic")
+	}
+	clientid, ok := ps["clientId"]
+	if !ok{
+		if uuid, err := uuid.NewUUID(); err != nil {
+			return nil, fmt.Errorf("mqtt sink fails to get uuid, the error is %s", err)
+		}else{
+			clientid = uuid.String()
+		}
+	}
+	ms := &MQTTSink{name:name, ruleId: ruleId, input: make(chan interface{}), srv: srv.(string), tpc: tpc.(string), clientid: clientid.(string)}
+	return ms, nil
+}
+
+func (ms *MQTTSink) GetName() string {
+	return ms.name
+}
+
+func (ms *MQTTSink) GetInput() (chan<- interface{}, string)  {
+	return ms.input, ms.name
+}
+
+func (ms *MQTTSink) Open(ctx context.Context, result chan<- error) {
+	log := common.GetLogger(ctx)
+	log.Printf("Opening mqtt sink for rule %s", ms.ruleId)
+
+	go func() {
+		exeCtx, cancel := context.WithCancel(ctx)
+		opts := MQTT.NewClientOptions().AddBroker(ms.srv).SetClientID(ms.clientid)
+
+		c := MQTT.NewClient(opts)
+		if token := c.Connect(); token.Wait() && token.Error() != nil {
+			result <- fmt.Errorf("Found error: %s", token.Error())
+			cancel()
+		}
+		log.Printf("The connection to server %s was established successfully", ms.srv)
+		ms.conn = c
+
+		for {
+			select {
+			case item := <-ms.input:
+				log.Infof("publish %s", item)
+				if token := c.Publish(ms.tpc, 0, false, item); token.Wait() && token.Error() != nil {
+					result <- fmt.Errorf("Publish error: %s", token.Error())
+				}
+
+			case <-exeCtx.Done():
+				c.Disconnect(5000)
+				log.Infof("Closing mqtt sink")
+				cancel()
+				return
+			}
+		}
+
+	}()
+}
+
+

xstream/streams.go

@@ -6,41 +6,48 @@ import (
 	"engine/xstream/operators"
 )
 
-var log = common.Log
-
 type TopologyNew struct {
 	sources []Source
 	sinks []Sink
 	ctx context.Context
-
+	cancel context.CancelFunc
 	drain chan error
 	ops []Operator
+	name string
 }
 
-func New() (*TopologyNew) {
-	tp := &TopologyNew{}
+func NewWithName(name string) *TopologyNew {
+	tp := &TopologyNew{name: name}
 	return tp
 }
 
-func (tp *TopologyNew) AddSrc(src Source) (*TopologyNew) {
-	tp.sources = append(tp.sources, src)
-	return tp
+func (s *TopologyNew) GetContext() context.Context {
+	return s.ctx
 }
 
-func (tp *TopologyNew) AddSink(inputs []Emitter, snk Sink) (*TopologyNew) {
+func (s *TopologyNew) Cancel(){
+	s.cancel()
+}
+
+func (s *TopologyNew) AddSrc(src Source) *TopologyNew {
+	s.sources = append(s.sources, src)
+	return s
+}
+
+func (s *TopologyNew) AddSink(inputs []Emitter, snk Sink) *TopologyNew {
 	for _, input := range inputs{
 		input.AddOutput(snk.GetInput())
 	}
-	tp.sinks = append(tp.sinks, snk)
-	return tp
+	s.sinks = append(s.sinks, snk)
+	return s
 }
 
-func (tp *TopologyNew) AddOperator(inputs []Emitter, operator Operator) (*TopologyNew) {
+func (s *TopologyNew) AddOperator(inputs []Emitter, operator Operator) *TopologyNew {
 	for _, input := range inputs{
 		input.AddOutput(operator.GetInput())
 	}
-	tp.ops = append(tp.ops, operator)
-	return tp
+	s.ops = append(s.ops, operator)
+	return s
 }
 
 func Transform(op operators.UnOperation, name string) *operators.UnaryOperator {
@@ -49,12 +56,13 @@ func Transform(op operators.UnOperation, name string) *operators.UnaryOperator {
 	return operator
 }
 
-func (tp *TopologyNew) Map(f interface{}) (*TopologyNew){
+func (s *TopologyNew) Map(f interface{}) *TopologyNew {
+	log := common.GetLogger(s.ctx)
 	op, err := MapFunc(f)
 	if err != nil {
 		log.Println(err)
 	}
-	return tp.Transform(op)
+	return s.Transform(op)
 }
 
 // Filter takes a predicate user-defined func that filters the stream.
@@ -82,8 +90,10 @@ func (s *TopologyNew) Transform(op operators.UnOperation) *TopologyNew {
 // prepareContext setups internal context before
 // stream starts execution.
 func (s *TopologyNew) prepareContext() {
-	if s.ctx == nil {
-		s.ctx = context.TODO()
+	if s.ctx == nil || s.ctx.Err() != nil {
+		s.ctx, s.cancel = context.WithCancel(context.Background())
+		contextLogger := common.Log.WithField("rule", s.name)
+		s.ctx = context.WithValue(s.ctx, common.LoggerKey, contextLogger)
 	}
 }
 
@@ -93,7 +103,7 @@ func (s *TopologyNew) drainErr(err error) {
 
 func (s *TopologyNew) Open() <-chan error {
 	s.prepareContext() // ensure context is set
-
+	log := common.GetLogger(s.ctx)
 	log.Println("Opening stream")
 
 	// open stream
@@ -102,6 +112,7 @@ func (s *TopologyNew) Open() <-chan error {
 		for _, src := range s.sources{
 			if err := src.Open(s.ctx); err != nil {
 				s.drainErr(err)
+				log.Println("Closing stream")
 				return
 			}
 		}
@@ -110,19 +121,24 @@ func (s *TopologyNew) Open() <-chan error {
 		for _, op := range s.ops {
 			if err := op.Exec(s.ctx); err != nil {
 				s.drainErr(err)
+				log.Println("Closing stream")
 				return
 			}
 		}
-
+		sinkErr := make(chan error)
+		defer func() {
+			log.Println("Closing sinkErr channel")
+			close(sinkErr)
+		}()
 		// open stream sink, after log sink is ready.
 		for _, snk := range s.sinks{
-			select {
-			case err := <-snk.Open(s.ctx):
-				log.Println("Closing stream")
-				s.drain <- err
-			}
+			snk.Open(s.ctx, sinkErr)
+		}
+		select {
+		case err := <- sinkErr:
+			log.Println("Closing stream")
+			s.drain <- err
 		}
-
 	}()
 
 	return s.drain

xstream/test/mock_sink.go

@@ -0,0 +1,53 @@
+package test
+
+import (
+	"context"
+	"engine/common"
+)
+
+type MockSink struct {
+	ruleId   string
+	name 	 string
+	results  [][]byte
+	input chan interface{}
+}
+
+func NewMockSink(name, ruleId string) *MockSink{
+	m := &MockSink{
+		ruleId:  ruleId,
+		name:    name,
+		input: make(chan interface{}),
+	}
+	return m
+}
+
+func (m *MockSink) Open(ctx context.Context, result chan<- error) {
+	log := common.GetLogger(ctx)
+	log.Trace("Opening mock sink")
+	m.results = make([][]byte, 0)
+	go func() {
+		for {
+			select {
+			case item := <-m.input:
+				if v, ok := item.([]byte); ok {
+					log.Infof("mock sink receive %s", item)
+					m.results = append(m.results, v)
+				}else{
+					log.Info("mock sink receive non byte data")
+				}
+
+			case <-ctx.Done():
+				log.Infof("mock sink %s done", m.name)
+				return
+			}
+		}
+	}()
+}
+
+func (m *MockSink) GetInput() (chan<- interface{}, string)  {
+	return m.input, m.name
+}
+
+func (m *MockSink) GetResults() [][]byte {
+	return m.results
+}

xstream/test/mock_source.go

@@ -0,0 +1,79 @@
+package test
+
+import (
+	"context"
+	"engine/common"
+	"engine/xsql"
+	"time"
+)
+
+type MockSource struct {
+	outs map[string]chan<- interface{}
+	data []*xsql.Tuple
+	name string
+	done chan<- struct{}
+	isEventTime bool
+}
+
+// New creates a new CsvSource
+func NewMockSource(data []*xsql.Tuple, name string, done chan<- struct{}, isEventTime bool) *MockSource {
+	mock := &MockSource{
+		data: data,
+		name: name,
+		outs: make(map[string]chan<- interface{}),
+		done: done,
+		isEventTime: isEventTime,
+	}
+	return mock
+}
+
+func (m *MockSource) Open(ctx context.Context) (err error) {
+	log := common.GetLogger(ctx)
+	log.Trace("Mocksource starts")
+	go func(){
+		for _, d := range m.data{
+			log.Infof("mock source is sending data %s", d)
+			if !m.isEventTime{
+				common.SetMockNow(d.Timestamp)
+				ticker := common.GetMockTicker()
+				timer := common.GetMockTimer()
+				if ticker != nil {
+					ticker.DoTick(d.Timestamp)
+				}
+				if timer != nil {
+					timer.DoTick(d.Timestamp)
+				}
+			}
+			for _, out := range m.outs{
+				select {
+				case out <- d:
+				case <-ctx.Done():
+					log.Trace("Mocksource stop")
+					return
+//				default:  TODO non blocking must have buffer?
+				}
+				time.Sleep(50 * time.Millisecond)
+			}
+			if m.isEventTime{
+				time.Sleep(1000 * time.Millisecond) //Let window run to make sure timers are set
+			}else{
+				time.Sleep(50 * time.Millisecond) //Let window run to make sure timers are set
+			}
+
+		}
+		if !m.isEventTime {
+			//reset now for the next test
+			common.SetMockNow(0)
+		}
+		m.done <- struct{}{}
+	}()
+	return nil
+}
+
+func (m *MockSource) AddOutput(output chan<- interface{}, name string) {
+	if _, ok := m.outs[name]; !ok{
+		m.outs[name] = output
+	}else{
+		common.Log.Warnf("fail to add output %s, operator %s already has an output of the same name", name, m.name)
+	}
+}

xstream/types.go

@@ -19,7 +19,7 @@ type Collector interface {
 
 type Sink interface {
 	Collector
-	Open(context context.Context) <-chan error
+	Open(context.Context, chan<- error)
 }
 
 type Operator interface{
@@ -30,4 +30,11 @@ type Operator interface{
 
 type TopNode interface{
 	GetName() string
+}
+
+type Rule struct{
+	Id string `json:"id"`
+	Sql string `json:"sql"`
+	Actions []map[string]interface{} `json:"actions"`
+	Options map[string]interface{} `json:"options"`
 }

xstream/util.go

@@ -2,6 +2,7 @@ package xstream
 
 import (
 	"encoding/json"
+	"engine/common"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -25,7 +26,7 @@ func GetConfAsString(file, key string) (string, error) {
 	} else if val == nil {
 		return "", nil
 	}else {
-		return "", fmt.Errorf("The value %s is not type of string for key %s.\n", val, key )
+		return "", fmt.Errorf("The value %s is not type of string for key %s", val, key )
 	}
 }
 
@@ -39,7 +40,7 @@ func GetConfAsInt(file, key string) (int, error) {
 	if v, ok := val.(float64); ok {
 		return int(v), nil
 	} else {
-		return 0, fmt.Errorf("The value {0} is not type of int for key {1}.\n", )
+		return 0, fmt.Errorf("The value {0} is not type of int for key {1}", )
 	}
 }
 
@@ -53,7 +54,7 @@ func GetConfAsFloat(file, key string) (float64, error) {
 	if v, ok := val.(float64); ok {
 		return v, nil
 	} else {
-		return 0, fmt.Errorf("The value {0} is not type of float for key {1}.\n", )
+		return 0, fmt.Errorf("The value {0} is not type of float for key {1}", )
 	}
 }
 
@@ -67,7 +68,7 @@ func GetConfAsBool(file, key string) (bool, error) {
 	if v, ok := val.(bool); ok {
 		return v, nil
 	} else {
-		return false, fmt.Errorf("The value {0} is not type of bool for key {1}.\n", )
+		return false, fmt.Errorf("The value {0} is not type of bool for key {1}", )
 	}
 }
 
@@ -95,7 +96,7 @@ func initConf(file string) (Conf, error) {
 		if err2 := json.Unmarshal([]byte(byteValue), &conf); err2 != nil {
 			return nil, err2
 		}
-		log.Printf("Successfully to load the configuration file %s.\n", fp)
+		common.Log.Printf("Successfully to load the configuration file %s", fp)
 	} else {
 		//Try as absolute path
 		if f, err1 := os.Open(file); err1 == nil {
@@ -103,9 +104,9 @@ func initConf(file string) (Conf, error) {
 			if err2 := json.Unmarshal([]byte(byteValue), &conf); err2 != nil {
 				return nil, err2
 			}
-			log.Printf("Successfully to load the configuration file %s.\n", file)
+			common.Log.Printf("Successfully to load the configuration file %s", file)
 		} else {
-			return nil, fmt.Errorf("Cannot load configuration file %s.\n", file)
+			return nil, fmt.Errorf("Cannot load configuration file %s", file)
 		}
 	}
 	return conf, nil