refactor(*): re-org the project structure

Signed-off-by: ngjaying <ngjaying@gmail.com>

xstream/cli/main.go

@@ -3,9 +3,10 @@ package main
 import (
 	"bufio"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/go-yaml/yaml"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/server"
 	"github.com/urfave/cli"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"net/rpc"
 	"os"
@@ -40,7 +41,7 @@ var (
 )
 
 func main() {
-	common.LoadFileType = LoadFileType
+	conf.LoadFileType = LoadFileType
 	app := cli.NewApp()
 	app.Version = Version
 
@@ -49,9 +50,9 @@ func main() {
 	//		Usage: "the name of stream",
 	//	}}
 
-	b, err := common.LoadConf(clientYaml)
+	b, err := conf.LoadConf(clientYaml)
 	if err != nil {
-		common.Log.Fatal(err)
+		conf.Log.Fatal(err)
 	}
 	var cfg map[string]clientConf
 	var config *clientConf
@@ -214,7 +215,7 @@ func main() {
 								}
 								rname := c.Args()[0]
 								var reply string
-								args := &common.RPCArgDesc{rname, string(rule)}
+								args := &server.RPCArgDesc{Name: rname, Json: string(rule)}
 								err = client.Call("Server.CreateRule", args, &reply)
 								if err != nil {
 									fmt.Println(err)
@@ -231,7 +232,7 @@ func main() {
 							rname := c.Args()[0]
 							rjson := c.Args()[1]
 							var reply string
-							args := &common.RPCArgDesc{rname, rjson}
+							args := &server.RPCArgDesc{Name: rname, Json: rjson}
 							err = client.Call("Server.CreateRule", args, &reply)
 							if err != nil {
 								fmt.Println(err)
@@ -264,8 +265,8 @@ func main() {
 						}
 						pname := c.Args()[1]
 						sfile := c.String("file")
-						args := &common.PluginDesc{
-							RPCArgDesc: common.RPCArgDesc{
+						args := &server.PluginDesc{
+							RPCArgDesc: server.RPCArgDesc{
 								Name: pname,
 							},
 							Type: ptype,
@@ -307,7 +308,7 @@ func main() {
 							return nil
 						}
 						var reply string
-						err = client.Call("Server.CreateService", &common.RPCArgDesc{
+						err = client.Call("Server.CreateService", &server.RPCArgDesc{
 							Name: c.Args()[0],
 							Json: c.Args()[1],
 						}, &reply)
@@ -378,8 +379,8 @@ func main() {
 							return nil
 						}
 						pname := c.Args()[1]
-						args := &common.PluginDesc{
-							RPCArgDesc: common.RPCArgDesc{
+						args := &server.PluginDesc{
+							RPCArgDesc: server.RPCArgDesc{
 								Name: pname,
 							},
 							Type: ptype,
@@ -524,8 +525,8 @@ func main() {
 							return nil
 						}
 						pname := c.Args()[1]
-						args := &common.PluginDesc{
-							RPCArgDesc: common.RPCArgDesc{
+						args := &server.PluginDesc{
+							RPCArgDesc: server.RPCArgDesc{
 								Name: pname,
 							},
 							Type: ptype,
@@ -828,8 +829,8 @@ func main() {
 						}
 						pname := c.Args()[1]
 						sfile := c.String("file")
-						args := &common.PluginDesc{
-							RPCArgDesc: common.RPCArgDesc{
+						args := &server.PluginDesc{
+							RPCArgDesc: server.RPCArgDesc{
 								Name: pname,
 							},
 						}

xstream/server/main.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/emqx/kuiper/xstream/server/server"
+import "github.com/emqx/kuiper/internal/server"
 
 var (
 	Version      = "unknown"

common/util.go

@@ -1,352 +0,0 @@
-package common
-
-import (
-	"archive/zip"
-	"encoding/json"
-	"fmt"
-	"github.com/benbjohnson/clock"
-	"github.com/emqx/kuiper/xstream/api"
-	"github.com/go-yaml/yaml"
-	"github.com/keepeye/logrus-filename"
-	"github.com/lestrrat-go/file-rotatelogs"
-	"github.com/sirupsen/logrus"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	"time"
-)
-
-const (
-	logFileName     = "stream.log"
-	etc_dir         = "etc"
-	data_dir        = "data"
-	log_dir         = "log"
-	plugins_dir     = "plugins"
-	StreamConf      = "kuiper.yaml"
-	KuiperBaseKey   = "KuiperBaseKey"
-	KuiperSyslogKey = "KuiperSyslogKey"
-	MetaKey         = "__meta"
-)
-
-var (
-	Log             *logrus.Logger
-	Config          *KuiperConf
-	IsTesting       bool
-	Clock           clock.Clock
-	logFile         *os.File
-	LoadFileType    = "relative"
-	AbsoluteMapping = map[string]string{
-		etc_dir:     "/etc/kuiper",
-		data_dir:    "/var/lib/kuiper/data",
-		log_dir:     "/var/log/kuiper",
-		plugins_dir: "/var/lib/kuiper/plugins",
-	}
-)
-
-func LoadConf(confName string) ([]byte, error) {
-	confDir, err := GetConfLoc()
-	if err != nil {
-		return nil, err
-	}
-
-	file := path.Join(confDir, confName)
-	b, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-type tlsConf struct {
-	Certfile string `yaml:"certfile"`
-	Keyfile  string `yaml:"keyfile"`
-}
-
-type KuiperConf struct {
-	Basic struct {
-		Debug          bool     `yaml:"debug"`
-		ConsoleLog     bool     `yaml:"consoleLog"`
-		FileLog        bool     `yaml:"fileLog"`
-		RotateTime     int      `yaml:"rotateTime"`
-		MaxAge         int      `yaml:"maxAge"`
-		Ip             string   `yaml:"ip"`
-		Port           int      `yaml:"port"`
-		RestIp         string   `yaml:"restIp"`
-		RestPort       int      `yaml:"restPort"`
-		RestTls        *tlsConf `yaml:"restTls"`
-		Prometheus     bool     `yaml:"prometheus"`
-		PrometheusPort int      `yaml:"prometheusPort"`
-		PluginHosts    string   `yaml:"pluginHosts"`
-	}
-	Rule api.RuleOption
-	Sink struct {
-		CacheThreshold    int  `yaml:"cacheThreshold"`
-		CacheTriggerCount int  `yaml:"cacheTriggerCount"`
-		DisableCache      bool `yaml:"disableCache""`
-	}
-}
-
-func init() {
-	Log = logrus.New()
-	initSyslog()
-	filenameHook := filename.NewHook()
-	filenameHook.Field = "file"
-	Log.AddHook(filenameHook)
-
-	Log.SetFormatter(&logrus.TextFormatter{
-		TimestampFormat: "2006-01-02 15:04:05",
-		DisableColors:   true,
-		FullTimestamp:   true,
-	})
-
-	Log.Debugf("init with args %s", os.Args)
-	for _, arg := range os.Args {
-		if strings.HasPrefix(arg, "-test.") {
-			IsTesting = true
-			break
-		}
-	}
-	if IsTesting {
-		Log.Debugf("running in testing mode")
-		Clock = clock.NewMock()
-	} else {
-		Clock = clock.New()
-	}
-}
-
-func InitConf() {
-	b, err := LoadConf(StreamConf)
-	if err != nil {
-		Log.Fatal(err)
-	}
-
-	kc := KuiperConf{
-		Rule: api.RuleOption{
-			LateTol:            1000,
-			Concurrency:        1,
-			BufferLength:       1024,
-			CheckpointInterval: 300000, //5 minutes
-			SendError:          true,
-		},
-	}
-	if err := yaml.Unmarshal(b, &kc); err != nil {
-		Log.Fatal(err)
-	} else {
-		Config = &kc
-	}
-	if 0 == len(Config.Basic.Ip) {
-		Config.Basic.Ip = "0.0.0.0"
-	}
-	if 0 == len(Config.Basic.RestIp) {
-		Config.Basic.RestIp = "0.0.0.0"
-	}
-
-	if Config.Basic.Debug {
-		Log.SetLevel(logrus.DebugLevel)
-	}
-
-	if Config.Basic.FileLog {
-		logDir, err := GetLoc(log_dir)
-		if err != nil {
-			Log.Fatal(err)
-		}
-
-		file := path.Join(logDir, logFileName)
-		logWriter, err := rotatelogs.New(
-			file+".%Y-%m-%d_%H-%M-%S",
-			rotatelogs.WithLinkName(file),
-			rotatelogs.WithRotationTime(time.Hour*time.Duration(Config.Basic.RotateTime)),
-			rotatelogs.WithMaxAge(time.Hour*time.Duration(Config.Basic.MaxAge)),
-		)
-
-		if err != nil {
-			fmt.Println("Failed to init log file settings..." + err.Error())
-			Log.Infof("Failed to log to file, using default stderr.")
-		} else if Config.Basic.ConsoleLog {
-			mw := io.MultiWriter(os.Stdout, logWriter)
-			Log.SetOutput(mw)
-		} else if !Config.Basic.ConsoleLog {
-			Log.SetOutput(logWriter)
-		}
-	} else if Config.Basic.ConsoleLog {
-		Log.SetOutput(os.Stdout)
-	}
-}
-
-func CloseLogger() {
-	if logFile != nil {
-		logFile.Close()
-	}
-}
-
-func GetConfLoc() (string, error) {
-	return GetLoc(etc_dir)
-}
-
-func GetDataLoc() (string, error) {
-	if IsTesting {
-		dataDir, err := GetLoc(data_dir)
-		if err != nil {
-			return "", err
-		}
-		d := path.Join(dataDir, "test")
-		if _, err := os.Stat(d); os.IsNotExist(err) {
-			err = os.MkdirAll(d, 0755)
-			if err != nil {
-				return "", err
-			}
-		}
-		return d, nil
-	}
-	return GetLoc(data_dir)
-}
-
-func GetPluginsLoc() (string, error) {
-	return GetLoc(plugins_dir)
-}
-
-func absolutePath(loc string) (dir string, err error) {
-	for relDir, absoluteDir := range AbsoluteMapping {
-		if strings.HasPrefix(loc, relDir) {
-			dir = strings.Replace(loc, relDir, absoluteDir, 1)
-			break
-		}
-	}
-	if 0 == len(dir) {
-		return "", fmt.Errorf("location %s is not allowed for absolue mode", loc)
-	}
-	return dir, nil
-}
-
-// GetLoc subdir must be a relative path
-func GetLoc(subdir string) (string, error) {
-	if "relative" == LoadFileType {
-		return relativePath(subdir)
-	}
-
-	if "absolute" == LoadFileType {
-		return absolutePath(subdir)
-	}
-	return "", fmt.Errorf("Unrecognized loading method.")
-}
-
-func relativePath(subdir string) (dir string, err error) {
-	dir, err = os.Getwd()
-	if err != nil {
-		return "", err
-	}
-
-	if base := os.Getenv(KuiperBaseKey); base != "" {
-		Log.Infof("Specified Kuiper base folder at location %s.\n", base)
-		dir = base
-	}
-	confDir := path.Join(dir, subdir)
-	if _, err := os.Stat(confDir); os.IsNotExist(err) {
-		lastdir := dir
-		for len(dir) > 0 {
-			dir = filepath.Dir(dir)
-			if lastdir == dir {
-				break
-			}
-			confDir = path.Join(dir, subdir)
-			if _, err := os.Stat(confDir); os.IsNotExist(err) {
-				lastdir = dir
-				continue
-			} else {
-				//Log.Printf("Trying to load file from %s", confDir)
-				return confDir, nil
-			}
-		}
-	} else {
-		//Log.Printf("Trying to load file from %s", confDir)
-		return confDir, nil
-	}
-
-	return "", fmt.Errorf("conf dir not found, please set KuiperBaseKey program environment variable correctly.")
-}
-
-func ProcessPath(p string) (string, error) {
-	if abs, err := filepath.Abs(p); err != nil {
-		return "", nil
-	} else {
-		if _, err := os.Stat(abs); os.IsNotExist(err) {
-			return "", err
-		}
-		return abs, nil
-	}
-}
-
-func ReadJsonUnmarshal(path string, ret interface{}) error {
-	sliByte, err := ioutil.ReadFile(path)
-	if nil != err {
-		return err
-	}
-	err = json.Unmarshal(sliByte, ret)
-	if nil != err {
-		return err
-	}
-	return nil
-}
-func WriteYamlMarshal(path string, data interface{}) error {
-	y, err := yaml.Marshal(data)
-	if nil != err {
-		return err
-	}
-	return ioutil.WriteFile(path, y, 0666)
-}
-
-func ReadYamlUnmarshal(path string, ret interface{}) error {
-	sliByte, err := ioutil.ReadFile(path)
-	if nil != err {
-		return err
-	}
-	err = yaml.Unmarshal(sliByte, ret)
-	if nil != err {
-		return err
-	}
-	return nil
-}
-
-func UnzipTo(f *zip.File, fpath string) error {
-	_, err := os.Stat(fpath)
-
-	if f.FileInfo().IsDir() {
-		// Make Folder
-		if _, err := os.Stat(fpath); os.IsNotExist(err) {
-			if err := os.MkdirAll(fpath, os.ModePerm); err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	if err == nil || !os.IsNotExist(err) {
-		if err = os.RemoveAll(fpath); err != nil {
-			return fmt.Errorf("failed to delete file %s", fpath)
-		}
-	}
-	if _, err := os.Stat(filepath.Dir(fpath)); os.IsNotExist(err) {
-		if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
-			return err
-		}
-	}
-
-	outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-	if err != nil {
-		return err
-	}
-
-	rc, err := f.Open()
-	if err != nil {
-		return err
-	}
-
-	_, err = io.Copy(outFile, rc)
-
-	outFile.Close()
-	rc.Close()
-	return err
-}

deploy/docker/conf_util.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/go-yaml/yaml"
+	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"strconv"
@@ -59,7 +59,7 @@ var file_keys_map = map[string]map[string]string{
 
 func main() {
 	fmt.Println(fileMap["edgex"])
-	files := make(map[string]map[interface{}]interface{})
+	files := make(map[string]map[string]interface{})
 	ProcessEnv(files, os.Environ())
 	for f, v := range files {
 		if bs, err := yaml.Marshal(v); err != nil {
@@ -78,14 +78,13 @@ func main() {
 	}
 }
 
-func printable(m map[interface{}]interface{}) map[interface{}]interface{} {
-	printableMap := make(map[interface{}]interface{})
+func printable(m map[string]interface{}) map[string]interface{} {
+	printableMap := make(map[string]interface{})
 	for k, v := range m {
-		ks, ok := k.(string)
-		if ok && strings.ToLower(ks) == "password" {
+		if strings.EqualFold(k, "password") {
 			printableMap[k] = "*"
 		} else {
-			if vm, ok := v.(map[interface{}]interface{}); ok {
+			if vm, ok := v.(map[string]interface{}); ok {
 				printableMap[k] = printable(vm)
 			} else {
 				printableMap[k] = v
@@ -95,13 +94,13 @@ func printable(m map[interface{}]interface{}) map[interface{}]interface{} {
 	return printableMap
 }
 
-func toPrintableString(m map[interface{}]interface{}) string {
+func toPrintableString(m map[string]interface{}) string {
 	p := printable(m)
 	b, _ := yaml.Marshal(p)
 	return string(b)
 }
 
-func ProcessEnv(files map[string]map[interface{}]interface{}, vars []string) {
+func ProcessEnv(files map[string]map[string]interface{}, vars []string) {
 	for _, e := range vars {
 		pair := strings.SplitN(e, "=", 2)
 		if len(pair) != 2 {
@@ -110,7 +109,7 @@ func ProcessEnv(files map[string]map[interface{}]interface{}, vars []string) {
 		}
 
 		valid := false
-		for k, _ := range fileMap {
+		for k := range fileMap {
 			if strings.HasPrefix(pair[0], strings.ToUpper(k)) {
 				valid = true
 				break
@@ -137,7 +136,7 @@ func ProcessEnv(files map[string]map[interface{}]interface{}, vars []string) {
 				if data, err := ioutil.ReadFile(fileMap[k]); err != nil {
 					fmt.Printf("%s\n", err)
 				} else {
-					m := make(map[interface{}]interface{})
+					m := make(map[string]interface{})
 					err = yaml.Unmarshal(data, &m)
 					if err != nil {
 						fmt.Println(err)
@@ -153,19 +152,19 @@ func ProcessEnv(files map[string]map[interface{}]interface{}, vars []string) {
 	}
 }
 
-func Handle(file string, conf map[interface{}]interface{}, skeys []string, val string) {
+func Handle(file string, conf map[string]interface{}, skeys []string, val string) {
 	key := getKey(file, skeys[0])
 	if len(skeys) == 1 {
 		conf[key] = getValueType(val)
 	} else if len(skeys) >= 2 {
 		if v, ok := conf[key]; ok {
-			if v1, ok1 := v.(map[interface{}]interface{}); ok1 {
+			if v1, ok1 := v.(map[string]interface{}); ok1 {
 				Handle(file, v1, skeys[1:], val)
 			} else {
 				fmt.Printf("Not expected map: %v\n", v)
 			}
 		} else {
-			v1 := make(map[interface{}]interface{})
+			v1 := make(map[string]interface{})
 			conf[key] = v1
 			Handle(file, v1, skeys[1:], val)
 		}

deploy/docker/conf_util_test.go

@@ -8,28 +8,28 @@ import (
 
 func TestHandle(t *testing.T) {
 	var tests = []struct {
-		config map[interface{}]interface{}
+		config map[string]interface{}
 		skeys  []string
 		val    string
-		exp    map[interface{}]interface{}
+		exp    map[string]interface{}
 	}{
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"default", "protocol"},
 			val:   "ssl",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "ssl",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
@@ -37,22 +37,22 @@ func TestHandle(t *testing.T) {
 		},
 
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"default", "optional", "CLIENTID"},
 			val:   "client2",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client2",
 					},
 				},
@@ -60,22 +60,22 @@ func TestHandle(t *testing.T) {
 		},
 
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"default", "optional", "KEEPALIVE"},
 			val:   "6000",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId":  "client1",
 						"KeepAlive": int64(6000),
 					},
@@ -84,22 +84,22 @@ func TestHandle(t *testing.T) {
 		},
 
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"default", "optional", "RETAINED"},
 			val:   "true",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 						"Retained": true,
 					},
@@ -108,22 +108,22 @@ func TestHandle(t *testing.T) {
 		},
 
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"default", "optional", "test"},
 			val:   "3.14",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 						"test":     3.14,
 					},
@@ -132,26 +132,26 @@ func TestHandle(t *testing.T) {
 		},
 
 		{
-			config: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			config: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
 			},
 			skeys: []string{"application_conf", "test"},
 			val:   "ssl",
-			exp: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			exp: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"port":     5563,
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "client1",
 					},
 				},
-				"application_conf": map[interface{}]interface{}{
+				"application_conf": map[string]interface{}{
 					"test": "ssl",
 				},
 			},
@@ -171,7 +171,7 @@ func TestProcessEnv(t *testing.T) {
 	var tests = []struct {
 		vars []string
 		file string
-		expt map[interface{}]interface{}
+		expt map[string]interface{}
 		out  string
 	}{
 		{
@@ -182,23 +182,23 @@ func TestProcessEnv(t *testing.T) {
 				"EDGEX__APPLICATION_CONF__PROTOCOL=ssl",
 			},
 			file: "edgex",
-			expt: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			expt: map[string]interface{}{
+				"default": map[string]interface{}{
 					"protocol": "tcp",
 					"type":     "zmq",
-					"optional": map[interface{}]interface{}{
+					"optional": map[string]interface{}{
 						"ClientId": "clientid_0000",
 						"Password": "should_not_print",
 					},
 				},
-				"application_conf": map[interface{}]interface{}{
+				"application_conf": map[string]interface{}{
 					"protocol": "ssl",
 				},
 			},
-			out: "application_conf:\n  protocol: ssl\ndefault:\n  optional:\n    ClientId: clientid_0000\n    Password: '*'\n  protocol: tcp\n  type: zmq\n",
+			out: "application_conf:\n    protocol: ssl\ndefault:\n    optional:\n        ClientId: clientid_0000\n        Password: '*'\n    protocol: tcp\n    type: zmq\n",
 		},
 	}
-	files := make(map[string]map[interface{}]interface{})
+	files := make(map[string]map[string]interface{})
 	for i, tt := range tests {
 		ProcessEnv(files, tt.vars)
 		if !reflect.DeepEqual(tt.expt, files[tt.file]) {
@@ -220,7 +220,7 @@ func TestProcessEnvArrayValue(t *testing.T) {
 	var tests = []struct {
 		vars []string
 		file string
-		expt map[interface{}]interface{}
+		expt map[string]interface{}
 	}{
 		{
 			vars: []string{
@@ -228,15 +228,15 @@ func TestProcessEnvArrayValue(t *testing.T) {
 				"MQTT_SOURCE__DEFAULT__TEST=[1,2]",
 			},
 			file: "mqtt_source",
-			expt: map[interface{}]interface{}{
-				"default": map[interface{}]interface{}{
+			expt: map[string]interface{}{
+				"default": map[string]interface{}{
 					"servers": []interface{}{"tcp://10.211.55.12:1883", "tcp://10.211.55.13:1883"},
 					"test":    []interface{}{int64(1), int64(2)},
 				},
 			},
 		},
 	}
-	files := make(map[string]map[interface{}]interface{})
+	files := make(map[string]map[string]interface{})
 	for i, tt := range tests {
 		ProcessEnv(files, tt.vars)
 		if !reflect.DeepEqual(tt.expt, files[tt.file]) {

etc/sources/file.yaml

@@ -7,4 +7,4 @@ default:
   interval: 0
 
 test:
-  path: fvt_scripts
+  path: test

examples/testExtension.go

@@ -1,51 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xsql/processors"
-	"github.com/emqx/kuiper/xstream/planner"
-	"path"
-	"time"
-)
-
-func main() {
-	log := common.Log
-	dbDir, err := common.GetDataLoc()
-	if err != nil {
-		log.Panic(err)
-	}
-	log.Infof("db location is %s", dbDir)
-
-	demo := `DROP STREAM ext`
-	processors.NewStreamProcessor(path.Join(dbDir, "stream")).ExecStmt(demo)
-
-	demo = "CREATE STREAM ext (count bigint) WITH (DATASOURCE=\"users\", FORMAT=\"JSON\", TYPE=\"random\")"
-	_, err = processors.NewStreamProcessor(path.Join(dbDir, "stream")).ExecStmt(demo)
-	if err != nil {
-		panic(err)
-	}
-
-	rp := processors.NewRuleProcessor(dbDir)
-	rp.ExecDrop("$$test1")
-	rs, err := rp.ExecCreate("$$test1", "{\"sql\": \"SELECT echo(count) FROM ext where count > 3\",\"actions\": [{\"memory\":  {}}]}")
-	if err != nil {
-		msg := fmt.Sprintf("failed to create rule: %s.", err)
-		log.Printf(msg)
-	}
-
-	tp, err := planner.Plan(rs, dbDir)
-	if err != nil {
-		log.Panicf("fail to init rule: %v", err)
-	}
-
-	go func() {
-		select {
-		case err := <-tp.Open():
-			log.Println(err)
-			tp.Cancel()
-		}
-	}()
-	time.Sleep(5000000 * time.Millisecond)
-	log.Infof("exit main program")
-}

extensions.mod

@@ -10,7 +10,6 @@ require (
 	github.com/edgexfoundry/go-mod-messaging/v2 v2.0.1
 	github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
 	github.com/gdexlab/go-render v1.0.1
-	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/protobuf v1.5.0
 	github.com/google/uuid v1.2.0
@@ -40,6 +39,7 @@ require (
 	google.golang.org/grpc v1.36.1
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/ini.v1 v1.62.0
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	github.com/emqx/kuiper/extensions v0.0.0
 )
 

extensions/functions/accumulateWordCount/accumulateWordCount.go

@@ -2,8 +2,8 @@ package main
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xsql"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/ast"
 	"strings"
 )
 
@@ -22,8 +22,8 @@ func (f *accumulateWordCountFunc) Validate(args []interface{}) error {
 	if len(args) != 2 {
 		return fmt.Errorf("wordCount function only supports 2 parameter but got %d", len(args))
 	}
-	if arg1, ok := args[1].(xsql.Expr); ok {
-		if _, ok := arg1.(*xsql.StringLiteral); !ok {
+	if arg1, ok := args[1].(ast.Expr); ok {
+		if _, ok := arg1.(*ast.StringLiteral); !ok {
 			return fmt.Errorf("the second parameter of wordCount function must be a string literal")
 		}
 	}

extensions/functions/countPlusOne/countPlusOne.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 )
 
 type countPlusOneFunc struct {

extensions/functions/echo/echo.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 )
 
 type echo struct {

extensions/functions/geohash/geohash.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"github.com/mmcloughlin/geohash"
 )
 

extensions/functions/image/resize.go

@@ -3,7 +3,7 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"github.com/nfnt/resize"
 	"image"
 	"image/jpeg"

extensions/functions/image/thumbnail.go

@@ -3,7 +3,7 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"github.com/nfnt/resize"
 	"image"
 	"image/jpeg"

extensions/functions/labelImage/labelImage.go

@@ -6,7 +6,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	tflite "github.com/mattn/go-tflite"
 	"github.com/nfnt/resize"
 	"image"

extensions/go.sum

@@ -3,10 +3,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
-github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
 github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
-github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
 github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -17,8 +18,6 @@ github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiU
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e h1:oJCXMss/3rg5F6Poy9wG3JQusc58Mzk5B9Z6wSnssNE=
-github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e/go.mod h1:errmMKH8tTB49UR2A8C8DPYkyudelsYJwJFaZHQ6ik8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -30,9 +29,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
-github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0-dev.100/go.mod h1:pfXURRetgIto0GR0sCjDrfa71hqJ1wxmQWi/mOzWfWU=
 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0/go.mod h1:pfXURRetgIto0GR0sCjDrfa71hqJ1wxmQWi/mOzWfWU=
-github.com/edgexfoundry/go-mod-messaging/v2 v2.0.0-dev.16/go.mod h1:bLKWB9yeOHLZoQtHLZlGwz8MjsMJIvHDFce7CcUb4fE=
+github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0/go.mod h1:pfXURRetgIto0GR0sCjDrfa71hqJ1wxmQWi/mOzWfWU=
+github.com/edgexfoundry/go-mod-messaging/v2 v2.0.1/go.mod h1:bLKWB9yeOHLZoQtHLZlGwz8MjsMJIvHDFce7CcUb4fE=
 github.com/edgexfoundry/go-mod-messaging/v2 v2.0.1/go.mod h1:bLKWB9yeOHLZoQtHLZlGwz8MjsMJIvHDFce7CcUb4fE=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -59,10 +58,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.6.1/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
 github.com/go-redis/redis/v7 v7.3.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -87,7 +84,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU=
@@ -130,7 +126,6 @@ github.com/lestrrat-go/strftime v1.0.3 h1:qqOPU7y+TM8Y803I8fG9c/DyKG3xH/xkng6keC
 github.com/lestrrat-go/strftime v1.0.3/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g=
 github.com/mattn/go-pointer v0.0.0-20190911064623-a0a44394634f h1:QTRRO+ozoYgT3CQRIzNVYJRU3DB8HRnkZv6mr4ISmMA=
 github.com/mattn/go-pointer v0.0.0-20190911064623-a0a44394634f/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc=
-github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
 github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mattn/go-tflite v1.0.1 h1:bTfbF7HIF0n3vQsl2JdMUhsFT/KkQuQlCy0UlnF9D4M=
 github.com/mattn/go-tflite v1.0.1/go.mod h1:LME9BQINAkZIOGDVDJJcCa2v0NMuV2AKaf1U47NVS4w=
@@ -295,7 +290,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

extensions/sinks/file/file.go

@@ -4,7 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"os"
 	"sync"
 	"time"

extensions/sinks/image/image.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"image/jpeg"
 	"image/png"
 	"io/ioutil"

extensions/sinks/influx/influx.go

@@ -4,7 +4,7 @@ package main
 
 import (
 	"encoding/json"
-	api "github.com/emqx/kuiper/xstream/api"
+	api "github.com/emqx/kuiper/pkg/api"
 	_ "github.com/influxdata/influxdb1-client/v2"
 	client "github.com/influxdata/influxdb1-client/v2"
 	"strings"

extensions/sinks/memory/memory.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/emqx/kuiper/xstream/api"
+import "github.com/emqx/kuiper/pkg/api"
 
 type memory struct {
 	results [][]byte

extensions/sinks/tdengine/tdengine.go

@@ -6,8 +6,9 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
 	_ "github.com/taosdata/driver-go/taosSql"
 	"reflect"
 	"strings"
@@ -102,21 +103,21 @@ func (this *taosConfig) buildSql(ctx api.StreamContext, mapData map[string]inter
 
 func (m *taosSink) Configure(props map[string]interface{}) error {
 	cfg := &taosConfig{}
-	err := common.MapToStruct(props, cfg)
+	err := cast.MapToStruct(props, cfg)
 	if err != nil {
 		return fmt.Errorf("read properties %v fail with error: %v", props, err)
 	}
 	if cfg.Ip == "" {
 		cfg.Ip = "127.0.0.1"
-		common.Log.Infof("Not find IP conf, will use default value '127.0.0.1'.")
+		conf.Log.Infof("Not find IP conf, will use default value '127.0.0.1'.")
 	}
 	if cfg.User == "" {
 		cfg.User = "root"
-		common.Log.Infof("Not find user conf, will use default value 'root'.")
+		conf.Log.Infof("Not find user conf, will use default value 'root'.")
 	}
 	if cfg.Password == "" {
 		cfg.Password = "taosdata"
-		common.Log.Infof("Not find password conf, will use default value 'taosdata'.")
+		conf.Log.Infof("Not find password conf, will use default value 'taosdata'.")
 	}
 	if cfg.Database == "" {
 		return fmt.Errorf("property database is required")

extensions/sinks/zmq/zmq.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	zmq "github.com/pebbe/zmq4"
 )
 

extensions/sources/random/random.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
+	"github.com/emqx/kuiper/pkg/message"
 	"math/rand"
 	"strings"
 	"time"
@@ -30,7 +31,7 @@ type randomSource struct {
 
 func (s *randomSource) Configure(topic string, props map[string]interface{}) error {
 	cfg := &randomSourceConfig{}
-	err := common.MapToStruct(props, cfg)
+	err := cast.MapToStruct(props, cfg)
 	if err != nil {
 		return fmt.Errorf("read properties %v fail with error: %v", props, err)
 	}
@@ -43,7 +44,7 @@ func (s *randomSource) Configure(topic string, props map[string]interface{}) err
 	if cfg.Interval <= 0 {
 		return fmt.Errorf("source `random` property `seed` must be a positive integer but got %d", cfg.Seed)
 	}
-	if strings.ToLower(cfg.Format) != common.FORMAT_JSON {
+	if strings.ToLower(cfg.Format) != message.FormatJson {
 		return fmt.Errorf("random source only supports `json` format")
 	}
 	s.conf = cfg
@@ -93,7 +94,7 @@ func randomize(p map[string]interface{}, seed int) map[string]interface{} {
 	r := make(map[string]interface{})
 	for k, v := range p {
 		//TODO other data types
-		vi, err := common.ToInt(v, common.STRICT)
+		vi, err := cast.ToInt(v, cast.STRICT)
 		if err != nil {
 			break
 		}

extensions/sources/zmq/zmq.go

@@ -3,8 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/message"
 	zmq "github.com/pebbe/zmq4"
 )
 
@@ -25,7 +25,7 @@ func (s *zmqSource) Configure(topic string, props map[string]interface{}) error
 	s.srv = srv.(string)
 	f, ok := props["format"]
 	if !ok {
-		s.messageFormat = common.FORMAT_JSON
+		s.messageFormat = message.FormatJson
 	} else {
 		s.messageFormat = f.(string)
 	}
@@ -66,7 +66,7 @@ func (s *zmqSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple,
 			if s.topic != "" {
 				meta["topic"] = string(msgs[0])
 			}
-			result, e := common.MessageDecode(m, s.messageFormat)
+			result, e := message.Decode(m, s.messageFormat)
 			if e != nil {
 				logger.Errorf("Invalid data format, cannot decode %v to %s format with error %s", m, s.messageFormat, e)
 			} else {

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/edgexfoundry/go-mod-messaging/v2 v2.0.1
 	github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect
 	github.com/gdexlab/go-render v1.0.1
-	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/protobuf v1.5.0
 	github.com/google/uuid v1.2.0
@@ -40,6 +39,7 @@ require (
 	google.golang.org/grpc v1.36.1
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/ini.v1 v1.62.0
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )
 
 go 1.16

go.sum

@@ -62,8 +62,6 @@ github.com/go-playground/validator/v10 v10.6.1/go.mod h1:xm76BBt941f7yWdGnI2DVPF
 github.com/go-redis/redis/v7 v7.3.0 h1:3oHqd0W7f/VLKBxeYTEpqdMUsmMectngjM9OtoRoIgg=
 github.com/go-redis/redis/v7 v7.3.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
-github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=

internal/conf/conf.go

@@ -0,0 +1,128 @@
+package conf
+
+import (
+	"fmt"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/lestrrat-go/file-rotatelogs"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
+)
+
+const StreamConf = "kuiper.yaml"
+
+var (
+	Config    *KuiperConf
+	IsTesting bool
+)
+
+func LoadConf(confName string) ([]byte, error) {
+	confDir, err := GetConfLoc()
+	if err != nil {
+		return nil, err
+	}
+
+	file := path.Join(confDir, confName)
+	b, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+type tlsConf struct {
+	Certfile string `yaml:"certfile"`
+	Keyfile  string `yaml:"keyfile"`
+}
+
+type KuiperConf struct {
+	Basic struct {
+		Debug          bool     `yaml:"debug"`
+		ConsoleLog     bool     `yaml:"consoleLog"`
+		FileLog        bool     `yaml:"fileLog"`
+		RotateTime     int      `yaml:"rotateTime"`
+		MaxAge         int      `yaml:"maxAge"`
+		Ip             string   `yaml:"ip"`
+		Port           int      `yaml:"port"`
+		RestIp         string   `yaml:"restIp"`
+		RestPort       int      `yaml:"restPort"`
+		RestTls        *tlsConf `yaml:"restTls"`
+		Prometheus     bool     `yaml:"prometheus"`
+		PrometheusPort int      `yaml:"prometheusPort"`
+		PluginHosts    string   `yaml:"pluginHosts"`
+	}
+	Rule api.RuleOption
+	Sink struct {
+		CacheThreshold    int  `yaml:"cacheThreshold"`
+		CacheTriggerCount int  `yaml:"cacheTriggerCount"`
+		DisableCache      bool `yaml:"disableCache""`
+	}
+}
+
+func InitConf() {
+	b, err := LoadConf(StreamConf)
+	if err != nil {
+		Log.Fatal(err)
+	}
+
+	kc := KuiperConf{
+		Rule: api.RuleOption{
+			LateTol:            1000,
+			Concurrency:        1,
+			BufferLength:       1024,
+			CheckpointInterval: 300000, //5 minutes
+			SendError:          true,
+		},
+	}
+	if err := yaml.Unmarshal(b, &kc); err != nil {
+		Log.Fatal(err)
+	} else {
+		Config = &kc
+	}
+	if 0 == len(Config.Basic.Ip) {
+		Config.Basic.Ip = "0.0.0.0"
+	}
+	if 0 == len(Config.Basic.RestIp) {
+		Config.Basic.RestIp = "0.0.0.0"
+	}
+
+	if Config.Basic.Debug {
+		Log.SetLevel(logrus.DebugLevel)
+	}
+
+	if Config.Basic.FileLog {
+		logDir, err := GetLoc(logDir)
+		if err != nil {
+			Log.Fatal(err)
+		}
+
+		file := path.Join(logDir, logFileName)
+		logWriter, err := rotatelogs.New(
+			file+".%Y-%m-%d_%H-%M-%S",
+			rotatelogs.WithLinkName(file),
+			rotatelogs.WithRotationTime(time.Hour*time.Duration(Config.Basic.RotateTime)),
+			rotatelogs.WithMaxAge(time.Hour*time.Duration(Config.Basic.MaxAge)),
+		)
+
+		if err != nil {
+			fmt.Println("Failed to init log file settings..." + err.Error())
+			Log.Infof("Failed to log to file, using default stderr.")
+		} else if Config.Basic.ConsoleLog {
+			mw := io.MultiWriter(os.Stdout, logWriter)
+			Log.SetOutput(mw)
+		} else if !Config.Basic.ConsoleLog {
+			Log.SetOutput(logWriter)
+		}
+	} else if Config.Basic.ConsoleLog {
+		Log.SetOutput(os.Stdout)
+	}
+}
+
+func init() {
+	InitLogger()
+	InitClock()
+}

internal/conf/logger.go

@@ -0,0 +1,45 @@
+package conf
+
+import (
+	filename "github.com/keepeye/logrus-filename"
+	"github.com/sirupsen/logrus"
+	"os"
+	"strings"
+)
+
+const (
+	logFileName = "stream.log"
+)
+
+var (
+	Log     *logrus.Logger
+	logFile *os.File
+)
+
+func InitLogger() {
+	Log = logrus.New()
+	initSyslog()
+	filenameHook := filename.NewHook()
+	filenameHook.Field = "file"
+	Log.AddHook(filenameHook)
+
+	Log.SetFormatter(&logrus.TextFormatter{
+		TimestampFormat: "2006-01-02 15:04:05",
+		DisableColors:   true,
+		FullTimestamp:   true,
+	})
+
+	Log.Debugf("init with args %s", os.Args)
+	for _, arg := range os.Args {
+		if strings.HasPrefix(arg, "-test.") {
+			IsTesting = true
+			break
+		}
+	}
+}
+
+func CloseLogger() {
+	if logFile != nil {
+		logFile.Close()
+	}
+}

internal/conf/path.go

@@ -0,0 +1,123 @@
+package conf
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	etcDir          = "etc"
+	dataDir         = "data"
+	logDir          = "log"
+	pluginsDir      = "plugins"
+	KuiperBaseKey   = "KuiperBaseKey"
+	KuiperSyslogKey = "KuiperSyslogKey"
+)
+
+var LoadFileType = "relative"
+var AbsoluteMapping = map[string]string{
+	etcDir:     "/etc/kuiper",
+	dataDir:    "/var/lib/kuiper/data",
+	logDir:     "/var/log/kuiper",
+	pluginsDir: "/var/lib/kuiper/plugins",
+}
+
+func GetConfLoc() (string, error) {
+	return GetLoc(etcDir)
+}
+
+func GetDataLoc() (string, error) {
+	if IsTesting {
+		dataDir, err := GetLoc(dataDir)
+		if err != nil {
+			return "", err
+		}
+		d := path.Join(dataDir, "test")
+		if _, err := os.Stat(d); os.IsNotExist(err) {
+			err = os.MkdirAll(d, 0755)
+			if err != nil {
+				return "", err
+			}
+		}
+		return d, nil
+	}
+	return GetLoc(dataDir)
+}
+
+func GetPluginsLoc() (string, error) {
+	return GetLoc(pluginsDir)
+}
+
+func absolutePath(loc string) (dir string, err error) {
+	for relDir, absoluteDir := range AbsoluteMapping {
+		if strings.HasPrefix(loc, relDir) {
+			dir = strings.Replace(loc, relDir, absoluteDir, 1)
+			break
+		}
+	}
+	if 0 == len(dir) {
+		return "", fmt.Errorf("location %s is not allowed for absolue mode", loc)
+	}
+	return dir, nil
+}
+
+// GetLoc subdir must be a relative path
+func GetLoc(subdir string) (string, error) {
+	if "relative" == LoadFileType {
+		return relativePath(subdir)
+	}
+
+	if "absolute" == LoadFileType {
+		return absolutePath(subdir)
+	}
+	return "", fmt.Errorf("Unrecognized loading method.")
+}
+
+func relativePath(subdir string) (dir string, err error) {
+	dir, err = os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	if base := os.Getenv(KuiperBaseKey); base != "" {
+		Log.Infof("Specified Kuiper base folder at location %s.\n", base)
+		dir = base
+	}
+	confDir := path.Join(dir, subdir)
+	if _, err := os.Stat(confDir); os.IsNotExist(err) {
+		lastdir := dir
+		for len(dir) > 0 {
+			dir = filepath.Dir(dir)
+			if lastdir == dir {
+				break
+			}
+			confDir = path.Join(dir, subdir)
+			if _, err := os.Stat(confDir); os.IsNotExist(err) {
+				lastdir = dir
+				continue
+			} else {
+				//Log.Printf("Trying to load file from %s", confDir)
+				return confDir, nil
+			}
+		}
+	} else {
+		//Log.Printf("Trying to load file from %s", confDir)
+		return confDir, nil
+	}
+
+	return "", fmt.Errorf("conf dir not found, please set KuiperBaseKey program environment variable correctly.")
+}
+
+func ProcessPath(p string) (string, error) {
+	if abs, err := filepath.Abs(p); err != nil {
+		return "", nil
+	} else {
+		if _, err := os.Stat(abs); os.IsNotExist(err) {
+			return "", err
+		}
+		return abs, nil
+	}
+}

common/util_test.go

@@ -1,47 +1,10 @@
-package common
+package conf
 
 import (
-	"reflect"
 	"strings"
 	"testing"
 )
 
-func TestMapConvert_Funcs(t *testing.T) {
-	source := map[interface{}]interface{}{
-		"QUERY_TABLE": "VBAP",
-		"ROWCOUNT":    10,
-		"FIELDS": []interface{}{
-			map[interface{}]interface{}{"FIELDNAME": "MANDT"},
-			map[interface{}]interface{}{"FIELDNAME": "VBELN"},
-			map[interface{}]interface{}{"FIELDNAME": "POSNR"},
-		},
-	}
-
-	exp := map[string]interface{}{
-		"QUERY_TABLE": "VBAP",
-		"ROWCOUNT":    10,
-		"FIELDS": []interface{}{
-			map[string]interface{}{"FIELDNAME": "MANDT"},
-			map[string]interface{}{"FIELDNAME": "VBELN"},
-			map[string]interface{}{"FIELDNAME": "POSNR"},
-		},
-	}
-
-	got := ConvertMap(source)
-	if !reflect.DeepEqual(exp, got) {
-		t.Errorf("result mismatch:\n\nexp=%s\n\ngot=%s\n\n", exp, got)
-	}
-}
-
-func TestGetDataLoc_Funcs(t *testing.T) {
-	d, err := GetDataLoc()
-	if err != nil {
-		t.Errorf("Errors when getting data loc: %s.", err)
-	} else if !strings.HasSuffix(d, "kuiper/data/test") {
-		t.Errorf("Unexpected data location %s", d)
-	}
-}
-
 func TestAbsolutePath(t *testing.T) {
 	var tests = []struct {
 		r string
@@ -54,7 +17,7 @@ func TestAbsolutePath(t *testing.T) {
 			r: "data/",
 			a: "/var/lib/kuiper/data/",
 		}, {
-			r: log_dir,
+			r: logDir,
 			a: "/var/log/kuiper",
 		}, {
 			r: "plugins",
@@ -72,3 +35,12 @@ func TestAbsolutePath(t *testing.T) {
 		}
 	}
 }
+
+func TestGetDataLoc_Funcs(t *testing.T) {
+	d, err := GetDataLoc()
+	if err != nil {
+		t.Errorf("Errors when getting data loc: %s.", err)
+	} else if !strings.HasSuffix(d, "kuiper/data/test") {
+		t.Errorf("Unexpected data location %s", d)
+	}
+}

common/syslog.go

@@ -1,6 +1,6 @@
 // +build !windows
 
-package common
+package conf
 
 import (
 	logrus_syslog "github.com/sirupsen/logrus/hooks/syslog"

common/syslog_win.go

@@ -1,6 +1,6 @@
 // +build windows
 
-package common
+package conf
 
 func initSyslog() {
 	// Not supported in windows, do nothing.

internal/conf/time.go

@@ -0,0 +1,31 @@
+package conf
+
+import (
+	"github.com/benbjohnson/clock"
+	"github.com/emqx/kuiper/pkg/cast"
+	"time"
+)
+
+var Clock clock.Clock
+
+func InitClock() {
+	if IsTesting {
+		Log.Debugf("running in testing mode")
+		Clock = clock.NewMock()
+	} else {
+		Clock = clock.New()
+	}
+}
+
+//Time related. For Mock
+func GetTicker(duration int) *clock.Ticker {
+	return Clock.Ticker(time.Duration(duration) * time.Millisecond)
+}
+
+func GetTimer(duration int) *clock.Timer {
+	return Clock.Timer(time.Duration(duration) * time.Millisecond)
+}
+
+func GetNowInMilli() int64 {
+	return cast.TimeToUnixMilli(Clock.Now())
+}

internal/conf/time_test.go

@@ -0,0 +1,12 @@
+package conf
+
+import (
+	"testing"
+)
+
+func TestMockClock(t *testing.T) {
+	n := GetNowInMilli()
+	if n != 0 {
+		t.Errorf("mock clock now mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", 0, n)
+	}
+}

internal/pkg/filex/parser.go

@@ -0,0 +1,38 @@
+package filex
+
+import (
+	"encoding/json"
+	"gopkg.in/yaml.v3"
+	"io/ioutil"
+)
+
+func ReadJsonUnmarshal(path string, ret interface{}) error {
+	sliByte, err := ioutil.ReadFile(path)
+	if nil != err {
+		return err
+	}
+	err = json.Unmarshal(sliByte, ret)
+	if nil != err {
+		return err
+	}
+	return nil
+}
+func WriteYamlMarshal(path string, data interface{}) error {
+	y, err := yaml.Marshal(data)
+	if nil != err {
+		return err
+	}
+	return ioutil.WriteFile(path, y, 0666)
+}
+
+func ReadYamlUnmarshal(path string, ret interface{}) error {
+	sliByte, err := ioutil.ReadFile(path)
+	if nil != err {
+		return err
+	}
+	err = yaml.Unmarshal(sliByte, ret)
+	if nil != err {
+		return err
+	}
+	return nil
+}

internal/pkg/filex/zip.go

@@ -0,0 +1,50 @@
+package filex
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func UnzipTo(f *zip.File, fpath string) error {
+	_, err := os.Stat(fpath)
+
+	if f.FileInfo().IsDir() {
+		// Make Folder
+		if _, err := os.Stat(fpath); os.IsNotExist(err) {
+			if err := os.MkdirAll(fpath, os.ModePerm); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	if err == nil || !os.IsNotExist(err) {
+		if err = os.RemoveAll(fpath); err != nil {
+			return fmt.Errorf("failed to delete file %s", fpath)
+		}
+	}
+	if _, err := os.Stat(filepath.Dir(fpath)); os.IsNotExist(err) {
+		if err := os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
+			return err
+		}
+	}
+
+	outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
+	if err != nil {
+		return err
+	}
+
+	rc, err := f.Open()
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(outFile, rc)
+
+	outFile.Close()
+	rc.Close()
+	return err
+}

common/http_util.go

@@ -1,11 +1,12 @@
-package common
+package httpx
 
 import (
 	"bytes"
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/pkg/api"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -119,7 +120,7 @@ func IsValidUrl(uri string) bool {
 }
 
 func DownloadFile(filepath string, uri string) error {
-	Log.Infof("Start to download file %s\n", uri)
+	conf.Log.Infof("Start to download file %s\n", uri)
 	u, err := url.ParseRequestURI(uri)
 	if err != nil {
 		return err
@@ -131,7 +132,7 @@ func DownloadFile(filepath string, uri string) error {
 		if strings.Index(u.Path, ":") == 2 {
 			u.Path = u.Path[1:]
 		}
-		Log.Debugf(u.Path)
+		conf.Log.Debugf(u.Path)
 		sourceFileStat, err := os.Stat(u.Path)
 		if err != nil {
 			return err

plugins/funcMeta.go

@@ -1,8 +1,9 @@
-package plugins
+package plugin
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
 	"io/ioutil"
 	"path"
 	"strings"
@@ -57,11 +58,11 @@ func newUiFuncs(fi *fileFuncs) *uiFuncs {
 	return uis
 }
 
-var g_funcMetadata map[string]*uiFuncs
+var gFuncmetadata map[string]*uiFuncs
 
 func (m *Manager) readFuncMetaDir() error {
-	g_funcMetadata = make(map[string]*uiFuncs)
-	confDir, err := common.GetConfLoc()
+	gFuncmetadata = make(map[string]*uiFuncs)
+	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err
 	}
@@ -85,7 +86,7 @@ func (m *Manager) readFuncMetaDir() error {
 }
 
 func (m *Manager) uninstalFunc(name string) {
-	if ui, ok := g_funcMetadata[name+".json"]; ok {
+	if ui, ok := gFuncmetadata[name+".json"]; ok {
 		if nil != ui.About {
 			ui.About.Installed = false
 		}
@@ -94,7 +95,7 @@ func (m *Manager) uninstalFunc(name string) {
 func (m *Manager) readFuncMetaFile(filePath string) error {
 	fiName := path.Base(filePath)
 	fis := new(fileFuncs)
-	err := common.ReadJsonUnmarshal(filePath, fis)
+	err := filex.ReadJsonUnmarshal(filePath, fis)
 	if nil != err {
 		return fmt.Errorf("filePath:%s err:%v", filePath, err)
 	}
@@ -105,12 +106,12 @@ func (m *Manager) readFuncMetaFile(filePath string) error {
 	} else {
 		_, fis.About.Installed = m.registry.Get(FUNCTION, strings.TrimSuffix(fiName, `.json`))
 	}
-	g_funcMetadata[fiName] = newUiFuncs(fis)
-	common.Log.Infof("funcMeta file : %s", fiName)
+	gFuncmetadata[fiName] = newUiFuncs(fis)
+	conf.Log.Infof("funcMeta file : %s", fiName)
 	return nil
 }
 func GetFunctions() (ret []*uiFuncs) {
-	for _, v := range g_funcMetadata {
+	for _, v := range gFuncmetadata {
 		ret = append(ret, v)
 	}
 	return ret

plugins/manager.go

@@ -1,13 +1,16 @@
-package plugins
+package plugin
 
 import (
 	"archive/zip"
 	"bytes"
 	"errors"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/common/kv"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
+	"github.com/emqx/kuiper/internal/pkg/httpx"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/errorx"
+	"github.com/emqx/kuiper/pkg/kv"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -207,17 +210,17 @@ func getPlugin(t string, pt PluginType) (plugin.Symbol, error) {
 		if err != nil {
 			return nil, fmt.Errorf("cannot get the plugin file path: %v", err)
 		}
-		common.Log.Debugf("Opening plugin %s", mod)
+		conf.Log.Debugf("Opening plugin %s", mod)
 		plug, err := plugin.Open(mod)
 		if err != nil {
 			return nil, fmt.Errorf("cannot open %s: %v", mod, err)
 		}
-		common.Log.Debugf("Successfully open plugin %s", mod)
+		conf.Log.Debugf("Successfully open plugin %s", mod)
 		nf, err = plug.Lookup(ut)
 		if err != nil {
 			return nil, fmt.Errorf("cannot find symbol %s, please check if it is exported", t)
 		}
-		common.Log.Debugf("Successfully look-up plugin %s", mod)
+		conf.Log.Debugf("Successfully look-up plugin %s", mod)
 		symbolRegistry[key] = nf
 	}
 	return nf, nil
@@ -267,17 +270,17 @@ type Manager struct {
 func NewPluginManager() (*Manager, error) {
 	var outerErr error
 	once.Do(func() {
-		dir, err := common.GetPluginsLoc()
+		dir, err := conf.GetPluginsLoc()
 		if err != nil {
 			outerErr = fmt.Errorf("cannot find plugins folder: %s", err)
 			return
 		}
-		etcDir, err := common.GetConfLoc()
+		etcDir, err := conf.GetConfLoc()
 		if err != nil {
 			outerErr = fmt.Errorf("cannot find etc folder: %s", err)
 			return
 		}
-		dbDir, err := common.GetDataLoc()
+		dbDir, err := conf.GetDataLoc()
 		if err != nil {
 			outerErr = fmt.Errorf("cannot find db folder: %s", err)
 			return
@@ -298,7 +301,7 @@ func NewPluginManager() (*Manager, error) {
 			plugins[i] = names
 		}
 		registry := &Registry{plugins: plugins, symbols: make(map[string]string)}
-		for pf, _ := range plugins[FUNCTION] {
+		for pf := range plugins[FUNCTION] {
 			l := make([]string, 0)
 			if ok, err := db.Get(pf, &l); ok {
 				registry.StoreSymbols(pf, l)
@@ -317,16 +320,16 @@ func NewPluginManager() (*Manager, error) {
 			db:        db,
 		}
 		if err := singleton.readSourceMetaDir(); nil != err {
-			common.Log.Errorf("readSourceMetaDir:%v", err)
+			conf.Log.Errorf("readSourceMetaDir:%v", err)
 		}
 		if err := singleton.readSinkMetaDir(); nil != err {
-			common.Log.Errorf("readSinkMetaDir:%v", err)
+			conf.Log.Errorf("readSinkMetaDir:%v", err)
 		}
 		if err := singleton.readFuncMetaDir(); nil != err {
-			common.Log.Errorf("readFuncMetaDir:%v", err)
+			conf.Log.Errorf("readFuncMetaDir:%v", err)
 		}
 		if err := singleton.readUiMsgDir(); nil != err {
-			common.Log.Errorf("readUiMsgDir:%v", err)
+			conf.Log.Errorf("readUiMsgDir:%v", err)
 		}
 	})
 	return singleton, outerErr
@@ -369,7 +372,7 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 	if name == "" {
 		return fmt.Errorf("invalid name %s: should not be empty", name)
 	}
-	if !common.IsValidUrl(uri) || !strings.HasSuffix(uri, ".zip") {
+	if !httpx.IsValidUrl(uri) || !strings.HasSuffix(uri, ".zip") {
 		return fmt.Errorf("invalid uri %s", uri)
 	}
 
@@ -406,7 +409,7 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 	//clean up: delete zip file and unzip files in error
 	defer os.Remove(zipPath)
 	//download
-	err = common.DownloadFile(zipPath, uri)
+	err = httpx.DownloadFile(zipPath, uri)
 	if err != nil {
 		return fmt.Errorf("fail to download file %s: %s", uri, err)
 	}
@@ -434,15 +437,15 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 	switch t {
 	case SINK:
 		if err := m.readSinkMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
-			common.Log.Errorf("readSinkFile:%v", err)
+			conf.Log.Errorf("readSinkFile:%v", err)
 		}
 	case SOURCE:
 		if err := m.readSourceMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
-			common.Log.Errorf("readSourceFile:%v", err)
+			conf.Log.Errorf("readSourceFile:%v", err)
 		}
 	case FUNCTION:
 		if err := m.readFuncMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
-			common.Log.Errorf("readFuncFile:%v", err)
+			conf.Log.Errorf("readFuncFile:%v", err)
 		}
 	}
 	return nil
@@ -609,12 +612,12 @@ func getSoFilePath(m *Manager, t PluginType, name string, isSoName bool) (string
 	} else {
 		soname, ok = m.registry.GetPluginBySymbol(t, name)
 		if !ok {
-			return "", common.NewErrorWithCode(common.NOT_FOUND, fmt.Sprintf("invalid symbol name %s: not exist", name))
+			return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid symbol name %s: not exist", name))
 		}
 	}
 	v, ok = m.registry.Get(t, soname)
 	if !ok {
-		return "", common.NewErrorWithCode(common.NOT_FOUND, fmt.Sprintf("invalid name %s: not exist", soname))
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid name %s: not exist", soname))
 	}
 
 	soFile := soname + ".so"
@@ -626,7 +629,7 @@ func getSoFilePath(m *Manager, t PluginType, name string, isSoName bool) (string
 		p = path.Join(m.pluginDir, PluginTypes[t], ucFirst(soFile))
 	}
 	if _, err := os.Stat(p); err != nil {
-		return "", common.NewErrorWithCode(common.NOT_FOUND, fmt.Sprintf("cannot find .so file for plugin %s", soname))
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("cannot find .so file for plugin %s", soname))
 	}
 	return p, nil
 }
@@ -654,7 +657,7 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 	for _, file := range r.File {
 		fileName := file.Name
 		if yamlFile == fileName {
-			err = common.UnzipTo(file, yamlPath)
+			err = filex.UnzipTo(file, yamlPath)
 			if err != nil {
 				return filenames, "", err
 			}
@@ -662,14 +665,14 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 			filenames = append(filenames, yamlPath)
 		} else if fileName == name+".json" {
 			jsonPath := path.Join(m.etcDir, PluginTypes[t], fileName)
-			if err := common.UnzipTo(file, jsonPath); nil != err {
-				common.Log.Errorf("Failed to decompress the metadata %s file", fileName)
+			if err := filex.UnzipTo(file, jsonPath); nil != err {
+				conf.Log.Errorf("Failed to decompress the metadata %s file", fileName)
 			} else {
 				revokeFiles = append(revokeFiles, jsonPath)
 			}
 		} else if soPrefix.Match([]byte(fileName)) {
 			soPath := path.Join(m.pluginDir, PluginTypes[t], fileName)
-			err = common.UnzipTo(file, soPath)
+			err = filex.UnzipTo(file, soPath)
 			if err != nil {
 				return filenames, "", err
 			}
@@ -677,12 +680,12 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 			revokeFiles = append(revokeFiles, soPath)
 			_, version = parseName(fileName)
 		} else if strings.HasPrefix(fileName, "etc/") {
-			err = common.UnzipTo(file, path.Join(m.etcDir, PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
+			err = filex.UnzipTo(file, path.Join(m.etcDir, PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
 			if err != nil {
 				return filenames, "", err
 			}
 		} else { //unzip other files
-			err = common.UnzipTo(file, path.Join(tempPath, fileName))
+			err = filex.UnzipTo(file, path.Join(tempPath, fileName))
 			if err != nil {
 				return filenames, "", err
 			}
@@ -711,11 +714,11 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 			for _, f := range revokeFiles {
 				os.RemoveAll(f)
 			}
-			common.Log.Infof(`err:%v stdout:%s stderr:%s`, err, outb.String(), errb.String())
+			conf.Log.Infof(`err:%v stdout:%s stderr:%s`, err, outb.String(), errb.String())
 			return filenames, "", err
 		} else {
-			common.Log.Infof(`run install script:%s`, outb.String())
-			common.Log.Infof("install %s plugin %s", PluginTypes[t], name)
+			conf.Log.Infof(`run install script:%s`, outb.String())
+			conf.Log.Infof("install %s plugin %s", PluginTypes[t], name)
 		}
 	}
 	return filenames, version, nil

plugins/manager_test.go

@@ -1,9 +1,9 @@
-package plugins
+package plugin
 
 import (
 	"errors"
 	"fmt"
-	"github.com/emqx/kuiper/xsql"
+	"github.com/emqx/kuiper/internal/xsql"
 	"net/http"
 	"net/http/httptest"
 	"os"

plugins/msg_util.go

@@ -1,17 +1,17 @@
-package plugins
+package plugin
 
 import (
-	"github.com/emqx/kuiper/common"
-	ini "gopkg.in/ini.v1"
+	kconf "github.com/emqx/kuiper/internal/conf"
+	"gopkg.in/ini.v1"
 	"io/ioutil"
 	"path"
 )
 
-var g_uiMsg map[string]*ini.File
+var gUimsg map[string]*ini.File
 
 func getMsg(language, section, key string) string {
 	language += ".ini"
-	if conf, ok := g_uiMsg[language]; ok {
+	if conf, ok := gUimsg[language]; ok {
 		s := conf.Section(section)
 		if s != nil {
 			return s.Key(key).String()
@@ -20,8 +20,8 @@ func getMsg(language, section, key string) string {
 	return ""
 }
 func (m *Manager) readUiMsgDir() error {
-	g_uiMsg = make(map[string]*ini.File)
-	confDir, err := common.GetConfLoc()
+	gUimsg = make(map[string]*ini.File)
+	confDir, err := kconf.GetConfLoc()
 	if nil != err {
 		return err
 	}
@@ -34,12 +34,12 @@ func (m *Manager) readUiMsgDir() error {
 
 	for _, info := range infos {
 		fName := info.Name()
-		common.Log.Infof("uiMsg file : %s", fName)
+		kconf.Log.Infof("uiMsg file : %s", fName)
 		fPath := path.Join(dir, fName)
 		if conf, err := ini.Load(fPath); nil != err {
 			return err
 		} else {
-			g_uiMsg[fName] = conf
+			gUimsg[fName] = conf
 		}
 	}
 	return nil

plugins/sinkMeta.go

@@ -1,8 +1,10 @@
-package plugins
+package plugin
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
+	"github.com/emqx/kuiper/pkg/cast"
 	"io/ioutil"
 	"path"
 	"strings"
@@ -112,7 +114,7 @@ func newField(fis []*fileField) (uis []field, err error) {
 		switch t := fi.Default.(type) {
 		case []map[string]interface{}:
 			var auxFi []*fileField
-			if err = common.MapToStruct(t, &auxFi); nil != err {
+			if err = cast.MapToStruct(t, &auxFi); nil != err {
 				return nil, err
 			}
 			if ui.Default, err = newField(auxFi); nil != err {
@@ -148,10 +150,10 @@ func newUiSink(fi *fileSink) (*uiSink, error) {
 	return ui, err
 }
 
-var g_sinkMetadata map[string]*uiSink //immutable
+var gSinkmetadata map[string]*uiSink //immutable
 func (m *Manager) readSinkMetaDir() error {
-	g_sinkMetadata = make(map[string]*uiSink)
-	confDir, err := common.GetConfLoc()
+	gSinkmetadata = make(map[string]*uiSink)
+	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err
 	}
@@ -176,7 +178,7 @@ func (m *Manager) readSinkMetaDir() error {
 }
 
 func (m *Manager) uninstalSink(name string) {
-	if ui, ok := g_sinkMetadata[name+".json"]; ok {
+	if ui, ok := gSinkmetadata[name+".json"]; ok {
 		if nil != ui.About {
 			ui.About.Installed = false
 		}
@@ -186,7 +188,7 @@ func (m *Manager) readSinkMetaFile(filePath string) error {
 	finame := path.Base(filePath)
 	pluginName := strings.TrimSuffix(finame, `.json`)
 	metadata := new(fileSink)
-	err := common.ReadJsonUnmarshal(filePath, metadata)
+	err := filex.ReadJsonUnmarshal(filePath, metadata)
 	if nil != err {
 		return fmt.Errorf("filePath:%s err:%v", filePath, err)
 	}
@@ -197,17 +199,17 @@ func (m *Manager) readSinkMetaFile(filePath string) error {
 	} else {
 		_, metadata.About.Installed = m.registry.Get(SINK, pluginName)
 	}
-	g_sinkMetadata[finame], err = newUiSink(metadata)
+	gSinkmetadata[finame], err = newUiSink(metadata)
 	if nil != err {
 		return err
 	}
-	common.Log.Infof("Loading metadata file for sink: %s", finame)
+	conf.Log.Infof("Loading metadata file for sink: %s", finame)
 	return nil
 }
 
 func GetSinkMeta(pluginName, language string) (*uiSink, error) {
 	fileName := pluginName + `.json`
-	sinkMetadata := g_sinkMetadata
+	sinkMetadata := gSinkmetadata
 	data, ok := sinkMetadata[fileName]
 	if !ok || data == nil {
 		return nil, fmt.Errorf(`%s%s`, getMsg(language, sink, "not_found_plugin"), pluginName)
@@ -221,7 +223,7 @@ type pluginfo struct {
 }
 
 func GetSinks() (sinks []*pluginfo) {
-	sinkMeta := g_sinkMetadata
+	sinkMeta := gSinkmetadata
 	for fileName, v := range sinkMeta {
 		node := new(pluginfo)
 		node.Name = strings.TrimSuffix(fileName, `.json`)

plugins/sinkMeta_test.go

@@ -1,4 +1,4 @@
-package plugins
+package plugin
 
 import (
 	"testing"
@@ -22,9 +22,9 @@ func TestHintWhenModifySink(t *testing.T) {
 		},
 	}
 
-	g_sinkMetadata = make(map[string]*uiSink)
-	g_sinkMetadata["taos.json"] = taosMeta
-	g_sinkMetadata["log.json"] = logMeta
+	gSinkmetadata = make(map[string]*uiSink)
+	gSinkmetadata["taos.json"] = taosMeta
+	gSinkmetadata["log.json"] = logMeta
 
 	oldSink, err := GetSinkMeta("taos", "en_US")
 	if err != nil {

plugins/sourceMeta.go

@@ -1,9 +1,11 @@
-package plugins
+package plugin
 
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
+	"github.com/emqx/kuiper/pkg/cast"
 	"io/ioutil"
 	"path"
 	"reflect"
@@ -55,10 +57,10 @@ func newUiSource(fi *fileSource) (*uiSource, error) {
 	return ui, nil
 }
 
-var g_sourceProperty map[string]*sourceProperty
+var gSourceproperty map[string]*sourceProperty
 
 func (m *Manager) uninstalSource(name string) {
-	if v, ok := g_sourceProperty[name+".json"]; ok {
+	if v, ok := gSourceproperty[name+".json"]; ok {
 		if ui := v.meta; nil != ui {
 			if nil != ui.About {
 				ui.About.Installed = false
@@ -72,7 +74,7 @@ func (m *Manager) readSourceMetaFile(filePath string) error {
 		fileName = "mqtt.json"
 	}
 	ptrMeta := new(fileSource)
-	err := common.ReadJsonUnmarshal(filePath, ptrMeta)
+	err := filex.ReadJsonUnmarshal(filePath, ptrMeta)
 	if nil != err || 0 == len(ptrMeta.ConfKeys) {
 		return fmt.Errorf("file:%s err:%v", filePath, err)
 	}
@@ -89,7 +91,7 @@ func (m *Manager) readSourceMetaFile(filePath string) error {
 
 	yamlData := make(map[string]map[string]interface{})
 	filePath = strings.TrimSuffix(filePath, `.json`) + `.yaml`
-	err = common.ReadYamlUnmarshal(filePath, &yamlData)
+	err = filex.ReadYamlUnmarshal(filePath, &yamlData)
 	if nil != err {
 		return fmt.Errorf("file:%s err:%v", filePath, err)
 	}
@@ -104,13 +106,13 @@ func (m *Manager) readSourceMetaFile(filePath string) error {
 		return err
 	}
 
-	g_sourceProperty[fileName] = property
+	gSourceproperty[fileName] = property
 	return err
 }
 
 func (m *Manager) readSourceMetaDir() error {
-	g_sourceProperty = make(map[string]*sourceProperty)
-	confDir, err := common.GetConfLoc()
+	gSourceproperty = make(map[string]*sourceProperty)
+	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err
 	}
@@ -132,22 +134,22 @@ func (m *Manager) readSourceMetaDir() error {
 			if err = m.readSourceMetaFile(filePath); nil != err {
 				return err
 			}
-			common.Log.Infof("sourceMeta file : %s", fileName)
+			conf.Log.Infof("sourceMeta file : %s", fileName)
 		}
 	}
 	return nil
 }
 
 func GetSourceConf(pluginName, language string) (b []byte, err error) {
-	property, ok := g_sourceProperty[pluginName+".json"]
+	property, ok := gSourceproperty[pluginName+".json"]
 	if ok {
 		cf := make(map[string]map[string]interface{})
 		for key, kvs := range property.cf {
-			aux := make(map[interface{}]interface{})
+			aux := make(map[string]interface{})
 			for k, v := range kvs {
 				aux[k] = v
 			}
-			cf[key] = common.ConvertMap(aux)
+			cf[key] = aux
 		}
 		if b, err = json.Marshal(cf); nil != err {
 			return nil, fmt.Errorf(`%s%v`, getMsg(language, source, "json_marshal_fail"), cf)
@@ -159,7 +161,7 @@ func GetSourceConf(pluginName, language string) (b []byte, err error) {
 }
 
 func GetSourceMeta(pluginName, language string) (ptrSourceProperty *uiSource, err error) {
-	property, ok := g_sourceProperty[pluginName+".json"]
+	property, ok := gSourceproperty[pluginName+".json"]
 	if ok {
 		return property.cfToMeta(language)
 	}
@@ -167,7 +169,7 @@ func GetSourceMeta(pluginName, language string) (ptrSourceProperty *uiSource, er
 }
 
 func GetSources() (sources []*pluginfo) {
-	for fileName, v := range g_sourceProperty {
+	for fileName, v := range gSourceproperty {
 		node := new(pluginfo)
 		node.Name = strings.TrimSuffix(fileName, `.json`)
 		if nil == v.meta {
@@ -194,7 +196,7 @@ func GetSources() (sources []*pluginfo) {
 }
 
 func GetSourceConfKeys(pluginName string) (keys []string) {
-	property := g_sourceProperty[pluginName+".json"]
+	property := gSourceproperty[pluginName+".json"]
 	if nil == property {
 		return keys
 	}
@@ -205,7 +207,7 @@ func GetSourceConfKeys(pluginName string) (keys []string) {
 }
 
 func DelSourceConfKey(pluginName, confKey, language string) error {
-	property := g_sourceProperty[pluginName+".json"]
+	property := gSourceproperty[pluginName+".json"]
 	if nil == property {
 		return fmt.Errorf(`%s%s`, getMsg(language, source, "not_found_plugin"), pluginName)
 	}
@@ -223,7 +225,7 @@ func AddSourceConfKey(pluginName, confKey, language string, content []byte) erro
 		return err
 	}
 
-	property := g_sourceProperty[pluginName+".json"]
+	property := gSourceproperty[pluginName+".json"]
 	if nil == property {
 		return fmt.Errorf(`%s%s`, getMsg(language, source, "not_found_plugin"), pluginName)
 	}
@@ -237,7 +239,7 @@ func AddSourceConfKey(pluginName, confKey, language string, content []byte) erro
 	}
 
 	property.cf[confKey] = reqField
-	g_sourceProperty[pluginName+".json"] = property
+	gSourceproperty[pluginName+".json"] = property
 	return property.saveCf(pluginName, language)
 }
 
@@ -248,7 +250,7 @@ func AddSourceConfKeyField(pluginName, confKey, language string, content []byte)
 		return err
 	}
 
-	property := g_sourceProperty[pluginName+".json"]
+	property := gSourceproperty[pluginName+".json"]
 	if nil == property {
 		return fmt.Errorf(`%s%s`, getMsg(language, source, "not_found_plugin"), pluginName)
 	}
@@ -281,7 +283,7 @@ func recursionDelMap(cf, fields map[string]interface{}, language string) error {
 			}
 
 			var auxCf map[string]interface{}
-			if err := common.MapToStruct(cf[k], &auxCf); nil != err {
+			if err := cast.MapToStruct(cf[k], &auxCf); nil != err {
 				return fmt.Errorf(`%s%s.%s`, getMsg(language, source, "type_conversion_fail"), k, delKey)
 			}
 			cf[k] = auxCf
@@ -290,11 +292,11 @@ func recursionDelMap(cf, fields map[string]interface{}, language string) error {
 		}
 		if reflect.Map == reflect.TypeOf(v).Kind() {
 			var auxCf, auxFields map[string]interface{}
-			if err := common.MapToStruct(cf[k], &auxCf); nil != err {
+			if err := cast.MapToStruct(cf[k], &auxCf); nil != err {
 				return fmt.Errorf(`%s%s.%v`, getMsg(language, source, "type_conversion_fail"), k, v)
 			}
 			cf[k] = auxCf
-			if err := common.MapToStruct(v, &auxFields); nil != err {
+			if err := cast.MapToStruct(v, &auxFields); nil != err {
 				return fmt.Errorf(`%s%s.%v`, getMsg(language, source, "type_conversion_fail"), k, v)
 			}
 			if err := recursionDelMap(auxCf, auxFields, language); nil != err {
@@ -312,7 +314,7 @@ func DelSourceConfKeyField(pluginName, confKey, language string, content []byte)
 		return err
 	}
 
-	property := g_sourceProperty[pluginName+".json"]
+	property := gSourceproperty[pluginName+".json"]
 	if nil == property {
 		return fmt.Errorf(`%s%s`, getMsg(language, source, "not_found_plugin"), pluginName)
 	}
@@ -351,13 +353,13 @@ func recursionNewFields(template []field, conf map[string]interface{}, ret *[]fi
 			if reflect.Map == reflect.TypeOf(v).Kind() {
 				var nextCf map[string]interface{}
 				if tmp, ok := v.(map[interface{}]interface{}); ok {
-					nextCf = common.ConvertMap(tmp)
+					nextCf = cast.ConvertMap(tmp)
 				} else {
-					if err := common.MapToStruct(v, &nextCf); nil != err {
+					if err := cast.MapToStruct(v, &nextCf); nil != err {
 						return err
 					}
 				}
-				if err := common.MapToStruct(template[i].Default, &auxTemplate); nil != err {
+				if err := cast.MapToStruct(template[i].Default, &auxTemplate); nil != err {
 					return err
 				}
 				if err := recursionNewFields(auxTemplate, nextCf, &auxRet); nil != err {
@@ -372,10 +374,10 @@ func recursionNewFields(template []field, conf map[string]interface{}, ret *[]fi
 	return nil
 }
 
-func (this *sourceProperty) cfToMeta(language string) (*uiSource, error) {
-	fields := this.meta.ConfKeys["default"]
+func (p *sourceProperty) cfToMeta(language string) (*uiSource, error) {
+	fields := p.meta.ConfKeys["default"]
 	ret := make(map[string][]field)
-	for k, kvs := range this.cf {
+	for k, kvs := range p.cf {
 		var sli []field
 		err := recursionNewFields(fields, kvs, &sli)
 		if nil != err {
@@ -384,13 +386,13 @@ func (this *sourceProperty) cfToMeta(language string) (*uiSource, error) {
 		ret[k] = sli
 	}
 	meta := new(uiSource)
-	*meta = *(this.meta)
+	*meta = *(p.meta)
 	meta.ConfKeys = ret
 	return meta, nil
 }
 
-func (this *sourceProperty) saveCf(pluginName, language string) error {
-	confDir, err := common.GetConfLoc()
+func (p *sourceProperty) saveCf(pluginName, language string) error {
+	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return fmt.Errorf(`%s%v`, getMsg(language, source, "not_found_file"), err)
 	}
@@ -401,7 +403,7 @@ func (this *sourceProperty) saveCf(pluginName, language string) error {
 		dir = confDir
 	}
 	filePath := path.Join(dir, pluginName+".yaml")
-	err = common.WriteYamlMarshal(filePath, this.cf)
+	err = filex.WriteYamlMarshal(filePath, p.cf)
 	if nil != err {
 		return fmt.Errorf(`%s%v`, getMsg(language, "source", "write_data_fail"), err)
 	}

internal/processor/rule.go

@@ -0,0 +1,292 @@
+package processor
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/topo"
+	"github.com/emqx/kuiper/internal/topo/nodes"
+	"github.com/emqx/kuiper/internal/topo/planner"
+	"github.com/emqx/kuiper/internal/xsql"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
+	"github.com/emqx/kuiper/pkg/errorx"
+	"github.com/emqx/kuiper/pkg/kv"
+	"os"
+	"path"
+)
+
+type RuleProcessor struct {
+	db        kv.KeyValue
+	rootDbDir string
+}
+
+func NewRuleProcessor(d string) *RuleProcessor {
+	processor := &RuleProcessor{
+		db:        kv.GetDefaultKVStore(path.Join(d, "rule")),
+		rootDbDir: d,
+	}
+	return processor
+}
+
+func (p *RuleProcessor) ExecCreate(name, ruleJson string) (*api.Rule, error) {
+	rule, err := p.getRuleByJson(name, ruleJson)
+	if err != nil {
+		return nil, err
+	}
+
+	err = p.db.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer p.db.Close()
+
+	err = p.db.Setnx(rule.Id, ruleJson)
+	if err != nil {
+		return nil, err
+	} else {
+		log.Infof("Rule %s is created.", rule.Id)
+	}
+
+	return rule, nil
+}
+func (p *RuleProcessor) ExecUpdate(name, ruleJson string) (*api.Rule, error) {
+	rule, err := p.getRuleByJson(name, ruleJson)
+	if err != nil {
+		return nil, err
+	}
+
+	err = p.db.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer p.db.Close()
+
+	err = p.db.Set(rule.Id, ruleJson)
+	if err != nil {
+		return nil, err
+	} else {
+		log.Infof("Rule %s is update.", rule.Id)
+	}
+
+	return rule, nil
+}
+
+func (p *RuleProcessor) ExecReplaceRuleState(name string, triggered bool) (err error) {
+	rule, err := p.GetRuleByName(name)
+	if err != nil {
+		return err
+	}
+
+	rule.Triggered = triggered
+	ruleJson, err := json.Marshal(rule)
+	if err != nil {
+		return fmt.Errorf("Marshal rule %s error : %s.", name, err)
+	}
+
+	err = p.db.Open()
+	if err != nil {
+		return err
+	}
+	defer p.db.Close()
+
+	err = p.db.Set(name, string(ruleJson))
+	if err != nil {
+		return err
+	} else {
+		log.Infof("Rule %s is replaced.", name)
+	}
+	return err
+}
+
+func (p *RuleProcessor) GetRuleByName(name string) (*api.Rule, error) {
+	err := p.db.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer p.db.Close()
+	var s1 string
+	f, _ := p.db.Get(name, &s1)
+	if !f {
+		return nil, errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("Rule %s is not found.", name))
+	}
+	return p.getRuleByJson(name, s1)
+}
+
+func (p *RuleProcessor) getDefaultRule(name, sql string) *api.Rule {
+	return &api.Rule{
+		Id:  name,
+		Sql: sql,
+		Options: &api.RuleOption{
+			IsEventTime:        false,
+			LateTol:            1000,
+			Concurrency:        1,
+			BufferLength:       1024,
+			SendMetaToSink:     false,
+			SendError:          true,
+			Qos:                api.AtMostOnce,
+			CheckpointInterval: 300000,
+		},
+	}
+}
+
+func (p *RuleProcessor) getRuleByJson(name, ruleJson string) (*api.Rule, error) {
+	opt := conf.Config.Rule
+	//set default rule options
+	rule := &api.Rule{
+		Options: &opt,
+	}
+	if err := json.Unmarshal([]byte(ruleJson), &rule); err != nil {
+		return nil, fmt.Errorf("Parse rule %s error : %s.", ruleJson, err)
+	}
+
+	//validation
+	if rule.Id == "" && name == "" {
+		return nil, fmt.Errorf("Missing rule id.")
+	}
+	if name != "" && rule.Id != "" && name != rule.Id {
+		return nil, fmt.Errorf("Name is not consistent with rule id.")
+	}
+	if rule.Id == "" {
+		rule.Id = name
+	}
+	if rule.Sql == "" {
+		return nil, fmt.Errorf("Missing rule SQL.")
+	}
+	if _, err := xsql.GetStatementFromSql(rule.Sql); err != nil {
+		return nil, err
+	}
+	if rule.Actions == nil || len(rule.Actions) == 0 {
+		return nil, fmt.Errorf("Missing rule actions.")
+	}
+	if rule.Options == nil {
+		rule.Options = &api.RuleOption{}
+	}
+	//Setnx default options
+	if rule.Options.CheckpointInterval < 0 {
+		return nil, fmt.Errorf("rule option checkpointInterval %d is invalid, require a positive integer", rule.Options.CheckpointInterval)
+	}
+	if rule.Options.Concurrency < 0 {
+		return nil, fmt.Errorf("rule option concurrency %d is invalid, require a positive integer", rule.Options.Concurrency)
+	}
+	if rule.Options.BufferLength < 0 {
+		return nil, fmt.Errorf("rule option bufferLength %d is invalid, require a positive integer", rule.Options.BufferLength)
+	}
+	if rule.Options.LateTol < 0 {
+		return nil, fmt.Errorf("rule option lateTolerance %d is invalid, require a positive integer", rule.Options.LateTol)
+	}
+	return rule, nil
+}
+
+func (p *RuleProcessor) ExecQuery(ruleid, sql string) (*topo.Topo, error) {
+	if tp, err := planner.PlanWithSourcesAndSinks(p.getDefaultRule(ruleid, sql), p.rootDbDir, nil, []*nodes.SinkNode{nodes.NewSinkNode("sink_memory_log", "logToMemory", nil)}); err != nil {
+		return nil, err
+	} else {
+		go func() {
+			select {
+			case err := <-tp.Open():
+				if err != nil {
+					log.Infof("closing query for error: %v", err)
+					tp.GetContext().SetError(err)
+					tp.Cancel()
+				} else {
+					log.Info("closing query")
+				}
+			}
+		}()
+		return tp, nil
+	}
+}
+
+func (p *RuleProcessor) ExecDesc(name string) (string, error) {
+	err := p.db.Open()
+	if err != nil {
+		return "", err
+	}
+	defer p.db.Close()
+	var s1 string
+	f, _ := p.db.Get(name, &s1)
+	if !f {
+		return "", fmt.Errorf("Rule %s is not found.", name)
+	}
+	dst := &bytes.Buffer{}
+	if err := json.Indent(dst, []byte(s1), "", "  "); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintln(dst.String()), nil
+}
+
+func (p *RuleProcessor) GetAllRules() ([]string, error) {
+	err := p.db.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer p.db.Close()
+	return p.db.Keys()
+}
+
+func (p *RuleProcessor) ExecDrop(name string) (string, error) {
+	err := p.db.Open()
+	if err != nil {
+		return "", err
+	}
+	defer p.db.Close()
+	result := fmt.Sprintf("Rule %s is dropped.", name)
+	var ruleJson string
+	if ok, _ := p.db.Get(name, &ruleJson); ok {
+		rule, err := p.getRuleByJson(name, ruleJson)
+		if err != nil {
+			return "", err
+		}
+		if err := cleanSinkCache(rule); err != nil {
+			result = fmt.Sprintf("%s. Clean sink cache faile: %s.", result, err)
+		}
+		if err := cleanCheckpoint(name); err != nil {
+			result = fmt.Sprintf("%s. Clean checkpoint cache faile: %s.", result, err)
+		}
+	}
+	err = p.db.Delete(name)
+	if err != nil {
+		return "", err
+	} else {
+		return result, nil
+	}
+}
+
+func cleanCheckpoint(name string) error {
+	dbDir, _ := conf.GetDataLoc()
+	c := path.Join(dbDir, "checkpoints", name)
+	return os.RemoveAll(c)
+}
+
+func cleanSinkCache(rule *api.Rule) error {
+	dbDir, err := conf.GetDataLoc()
+	if err != nil {
+		return err
+	}
+	store := kv.GetDefaultKVStore(path.Join(dbDir, "sink"))
+	err = store.Open()
+	if err != nil {
+		return err
+	}
+	defer store.Close()
+	for d, m := range rule.Actions {
+		con := 1
+		for name, action := range m {
+			props, _ := action.(map[string]interface{})
+			if c, ok := props["concurrency"]; ok {
+				if t, err := cast.ToInt(c, cast.STRICT); err == nil && t > 0 {
+					con = t
+				}
+			}
+			for i := 0; i < con; i++ {
+				key := fmt.Sprintf("%s%s_%d%d", rule.Id, name, d, i)
+				conf.Log.Debugf("delete cache key %s", key)
+				store.Delete(key)
+			}
+		}
+	}
+	return nil
+}

xsql/processors/simple_processor_test.go

@@ -1,7 +1,7 @@
-package processors
+package processor
 
 import (
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 	"reflect"
 	"testing"
 )

internal/processor/stream.go

@@ -0,0 +1,315 @@
+package processor
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/xsql"
+	"github.com/emqx/kuiper/pkg/ast"
+	"github.com/emqx/kuiper/pkg/errorx"
+	"github.com/emqx/kuiper/pkg/kv"
+	"strings"
+)
+
+var (
+	log = conf.Log
+)
+
+type StreamProcessor struct {
+	db kv.KeyValue
+}
+
+//@params d : the directory of the DB to save the stream info
+func NewStreamProcessor(d string) *StreamProcessor {
+	processor := &StreamProcessor{
+		db: kv.GetDefaultKVStore(d),
+	}
+	return processor
+}
+
+func (p *StreamProcessor) ExecStmt(statement string) (result []string, err error) {
+	parser := xsql.NewParser(strings.NewReader(statement))
+	stmt, err := xsql.Language.Parse(parser)
+	if err != nil {
+		return nil, err
+	}
+	switch s := stmt.(type) {
+	case *ast.StreamStmt: //Table is also StreamStmt
+		var r string
+		err = p.execSave(s, statement, false)
+		stt := ast.StreamTypeMap[s.StreamType]
+		if err != nil {
+			err = fmt.Errorf("Create %s fails: %v.", stt, err)
+		} else {
+			r = fmt.Sprintf("%s %s is created.", strings.Title(stt), s.Name)
+			log.Printf("%s", r)
+		}
+		result = append(result, r)
+	case *ast.ShowStreamsStatement:
+		result, err = p.execShow(ast.TypeStream)
+	case *ast.ShowTablesStatement:
+		result, err = p.execShow(ast.TypeTable)
+	case *ast.DescribeStreamStatement:
+		var r string
+		r, err = p.execDescribe(s, ast.TypeStream)
+		result = append(result, r)
+	case *ast.DescribeTableStatement:
+		var r string
+		r, err = p.execDescribe(s, ast.TypeTable)
+		result = append(result, r)
+	case *ast.ExplainStreamStatement:
+		var r string
+		r, err = p.execExplain(s, ast.TypeStream)
+		result = append(result, r)
+	case *ast.ExplainTableStatement:
+		var r string
+		r, err = p.execExplain(s, ast.TypeTable)
+		result = append(result, r)
+	case *ast.DropStreamStatement:
+		var r string
+		r, err = p.execDrop(s, ast.TypeStream)
+		result = append(result, r)
+	case *ast.DropTableStatement:
+		var r string
+		r, err = p.execDrop(s, ast.TypeTable)
+		result = append(result, r)
+	default:
+		return nil, fmt.Errorf("Invalid stream statement: %s", statement)
+	}
+
+	return
+}
+
+func (p *StreamProcessor) execSave(stmt *ast.StreamStmt, statement string, replace bool) error {
+	err := p.db.Open()
+	if err != nil {
+		return fmt.Errorf("error when opening db: %v.", err)
+	}
+	defer p.db.Close()
+	s, err := json.Marshal(xsql.StreamInfo{
+		StreamType: stmt.StreamType,
+		Statement:  statement,
+	})
+	if err != nil {
+		return fmt.Errorf("error when saving to db: %v.", err)
+	}
+	if replace {
+		err = p.db.Set(string(stmt.Name), string(s))
+	} else {
+		err = p.db.Setnx(string(stmt.Name), string(s))
+	}
+	return err
+}
+
+func (p *StreamProcessor) ExecReplaceStream(statement string, st ast.StreamType) (string, error) {
+	parser := xsql.NewParser(strings.NewReader(statement))
+	stmt, err := xsql.Language.Parse(parser)
+	if err != nil {
+		return "", err
+	}
+	stt := ast.StreamTypeMap[st]
+	switch s := stmt.(type) {
+	case *ast.StreamStmt:
+		if s.StreamType != st {
+			return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("%s %s is not found", ast.StreamTypeMap[st], s.Name))
+		}
+		err = p.execSave(s, statement, true)
+		if err != nil {
+			return "", fmt.Errorf("Replace %s fails: %v.", stt, err)
+		} else {
+			info := fmt.Sprintf("%s %s is replaced.", strings.Title(stt), s.Name)
+			log.Printf("%s", info)
+			return info, nil
+		}
+	default:
+		return "", fmt.Errorf("Invalid %s statement: %s", stt, statement)
+	}
+}
+
+func (p *StreamProcessor) ExecStreamSql(statement string) (string, error) {
+	r, err := p.ExecStmt(statement)
+	if err != nil {
+		return "", err
+	} else {
+		return strings.Join(r, "\n"), err
+	}
+}
+
+func (p *StreamProcessor) execShow(st ast.StreamType) ([]string, error) {
+	keys, err := p.ShowStream(st)
+	if len(keys) == 0 {
+		keys = append(keys, fmt.Sprintf("No %s definitions are found.", ast.StreamTypeMap[st]))
+	}
+	return keys, err
+}
+
+func (p *StreamProcessor) ShowStream(st ast.StreamType) ([]string, error) {
+	stt := ast.StreamTypeMap[st]
+	err := p.db.Open()
+	if err != nil {
+		return nil, fmt.Errorf("Show %ss fails, error when opening db: %v.", stt, err)
+	}
+	defer p.db.Close()
+	keys, err := p.db.Keys()
+	if err != nil {
+		return nil, fmt.Errorf("Show %ss fails, error when loading data from db: %v.", stt, err)
+	}
+	var (
+		v      string
+		vs     = &xsql.StreamInfo{}
+		result = make([]string, 0)
+	)
+	for _, k := range keys {
+		if ok, _ := p.db.Get(k, &v); ok {
+			if err := json.Unmarshal([]byte(v), vs); err == nil && vs.StreamType == st {
+				result = append(result, k)
+			}
+		}
+	}
+	return result, nil
+}
+
+func (p *StreamProcessor) getStream(name string, st ast.StreamType) (string, error) {
+	vs, err := xsql.GetDataSourceStatement(p.db, name)
+	if vs != nil && vs.StreamType == st {
+		return vs.Statement, nil
+	}
+	if err != nil {
+		return "", err
+	}
+	return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("%s %s is not found", ast.StreamTypeMap[st], name))
+}
+
+func (p *StreamProcessor) execDescribe(stmt ast.NameNode, st ast.StreamType) (string, error) {
+	streamStmt, err := p.DescStream(stmt.GetName(), st)
+	if err != nil {
+		return "", err
+	}
+	switch s := streamStmt.(type) {
+	case *ast.StreamStmt:
+		var buff bytes.Buffer
+		buff.WriteString("Fields\n--------------------------------------------------------------------------------\n")
+		for _, f := range s.StreamFields {
+			buff.WriteString(f.Name + "\t")
+			buff.WriteString(printFieldType(f.FieldType))
+			buff.WriteString("\n")
+		}
+		buff.WriteString("\n")
+		printOptions(s.Options, &buff)
+		return buff.String(), err
+	default:
+		return "%s", fmt.Errorf("Error resolving the %s %s, the data in db may be corrupted.", ast.StreamTypeMap[st], stmt.GetName())
+	}
+
+}
+
+func printOptions(opts *ast.Options, buff *bytes.Buffer) {
+	if opts.CONF_KEY != "" {
+		buff.WriteString(fmt.Sprintf("CONF_KEY: %s\n", opts.CONF_KEY))
+	}
+	if opts.DATASOURCE != "" {
+		buff.WriteString(fmt.Sprintf("DATASOURCE: %s\n", opts.DATASOURCE))
+	}
+	if opts.FORMAT != "" {
+		buff.WriteString(fmt.Sprintf("FORMAT: %s\n", opts.FORMAT))
+	}
+	if opts.KEY != "" {
+		buff.WriteString(fmt.Sprintf("KEY: %s\n", opts.KEY))
+	}
+	if opts.RETAIN_SIZE != 0 {
+		buff.WriteString(fmt.Sprintf("RETAIN_SIZE: %d\n", opts.RETAIN_SIZE))
+	}
+	if opts.SHARED {
+		buff.WriteString(fmt.Sprintf("SHARED: %v\n", opts.SHARED))
+	}
+	if opts.STRICT_VALIDATION {
+		buff.WriteString(fmt.Sprintf("STRICT_VALIDATION: %v\n", opts.STRICT_VALIDATION))
+	}
+	if opts.TIMESTAMP != "" {
+		buff.WriteString(fmt.Sprintf("TIMESTAMP: %s\n", opts.TIMESTAMP))
+	}
+	if opts.TIMESTAMP_FORMAT != "" {
+		buff.WriteString(fmt.Sprintf("TIMESTAMP_FORMAT: %s\n", opts.TIMESTAMP_FORMAT))
+	}
+	if opts.TYPE != "" {
+		buff.WriteString(fmt.Sprintf("TYPE: %s\n", opts.TYPE))
+	}
+}
+
+func (p *StreamProcessor) DescStream(name string, st ast.StreamType) (ast.Statement, error) {
+	statement, err := p.getStream(name, st)
+	if err != nil {
+		return nil, fmt.Errorf("Describe %s fails, %s.", ast.StreamTypeMap[st], err)
+	}
+	parser := xsql.NewParser(strings.NewReader(statement))
+	stream, err := xsql.Language.Parse(parser)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (p *StreamProcessor) execExplain(stmt ast.NameNode, st ast.StreamType) (string, error) {
+	_, err := p.getStream(stmt.GetName(), st)
+	if err != nil {
+		return "", fmt.Errorf("Explain %s fails, %s.", ast.StreamTypeMap[st], err)
+	}
+	return "TO BE SUPPORTED", nil
+}
+
+func (p *StreamProcessor) execDrop(stmt ast.NameNode, st ast.StreamType) (string, error) {
+	s, err := p.DropStream(stmt.GetName(), st)
+	if err != nil {
+		return s, fmt.Errorf("Drop %s fails: %s.", ast.StreamTypeMap[st], err)
+	}
+	return s, nil
+}
+
+func (p *StreamProcessor) DropStream(name string, st ast.StreamType) (string, error) {
+	defer p.db.Close()
+	_, err := p.getStream(name, st)
+	if err != nil {
+		return "", err
+	}
+
+	err = p.db.Open()
+	if err != nil {
+		return "", fmt.Errorf("error when opening db: %v", err)
+	}
+	defer p.db.Close()
+	err = p.db.Delete(name)
+	if err != nil {
+		return "", err
+	} else {
+		return fmt.Sprintf("%s %s is dropped.", strings.Title(ast.StreamTypeMap[st]), name), nil
+	}
+}
+
+func printFieldType(ft ast.FieldType) (result string) {
+	switch t := ft.(type) {
+	case *ast.BasicType:
+		result = t.Type.String()
+	case *ast.ArrayType:
+		result = "array("
+		if t.FieldType != nil {
+			result += printFieldType(t.FieldType)
+		} else {
+			result += t.Type.String()
+		}
+		result += ")"
+	case *ast.RecType:
+		result = "struct("
+		isFirst := true
+		for _, f := range t.StreamFields {
+			if isFirst {
+				isFirst = false
+			} else {
+				result += ", "
+			}
+			result = result + f.Name + " " + printFieldType(f.FieldType)
+		}
+		result += ")"
+	}
+	return
+}

xsql/processors/stream_processor_test.go

@@ -1,15 +1,15 @@
-package processors
+package processor
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/testx"
 	"path"
 	"reflect"
 	"testing"
 )
 
 var (
-	DbDir = common.GetDbDir()
+	DbDir = testx.GetDbDir()
 )
 
 func TestStreamCreateProcessor(t *testing.T) {
@@ -91,7 +91,7 @@ func TestStreamCreateProcessor(t *testing.T) {
 	streamDB := path.Join(DbDir, "streamTest")
 	for i, tt := range tests {
 		results, err := NewStreamProcessor(streamDB).ExecStmt(tt.s)
-		if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
 		} else if tt.err == "" {
 			if !reflect.DeepEqual(tt.r, results) {
@@ -177,10 +177,10 @@ func TestTableProcessor(t *testing.T) {
 
 	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
 
-	streamDB := path.Join(common.GetDbDir(), "streamTest")
+	streamDB := path.Join(testx.GetDbDir(), "streamTest")
 	for i, tt := range tests {
 		results, err := NewStreamProcessor(streamDB).ExecStmt(tt.s)
-		if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
 		} else if tt.err == "" {
 			if !reflect.DeepEqual(tt.r, results) {

common/data.go

@@ -1,4 +1,4 @@
-package common
+package server
 
 type RPCArgDesc struct {
 	Name, Json string

common/os_util.go

@@ -1,4 +1,4 @@
-package common
+package server
 
 import (
 	"bufio"

xstream/server/server/rest.go

@@ -4,11 +4,12 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/plugins"
-	"github.com/emqx/kuiper/services"
-	"github.com/emqx/kuiper/xsql"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/plugin"
+	"github.com/emqx/kuiper/internal/service"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/ast"
+	"github.com/emqx/kuiper/pkg/errorx"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"golang.org/x/net/html"
@@ -50,9 +51,9 @@ func handleError(w http.ResponseWriter, err error, prefix string, logger api.Log
 	logger.Error(message)
 	var ec int
 	switch e := err.(type) {
-	case *common.Error:
+	case *errorx.Error:
 		switch e.Code() {
-		case common.NOT_FOUND:
+		case errorx.NOT_FOUND:
 			ec = http.StatusNotFound
 		default:
 			ec = http.StatusBadRequest
@@ -157,13 +158,13 @@ func pingHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
-func sourcesManageHandler(w http.ResponseWriter, r *http.Request, st xsql.StreamType) {
+func sourcesManageHandler(w http.ResponseWriter, r *http.Request, st ast.StreamType) {
 	defer r.Body.Close()
 	switch r.Method {
 	case http.MethodGet:
 		content, err := streamProcessor.ShowStream(st)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(xsql.StreamTypeMap[st])), logger)
+			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
 			return
 		}
 		jsonResponse(content, w, logger)
@@ -175,7 +176,7 @@ func sourcesManageHandler(w http.ResponseWriter, r *http.Request, st xsql.Stream
 		}
 		content, err := streamProcessor.ExecStreamSql(v.Sql)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(xsql.StreamTypeMap[st])), logger)
+			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
 			return
 		}
 		w.WriteHeader(http.StatusCreated)
@@ -183,7 +184,7 @@ func sourcesManageHandler(w http.ResponseWriter, r *http.Request, st xsql.Stream
 	}
 }
 
-func sourceManageHandler(w http.ResponseWriter, r *http.Request, st xsql.StreamType) {
+func sourceManageHandler(w http.ResponseWriter, r *http.Request, st ast.StreamType) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
@@ -192,14 +193,14 @@ func sourceManageHandler(w http.ResponseWriter, r *http.Request, st xsql.StreamT
 	case http.MethodGet:
 		content, err := streamProcessor.DescStream(name, st)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("describe %s error", xsql.StreamTypeMap[st]), logger)
+			handleError(w, err, fmt.Sprintf("describe %s error", ast.StreamTypeMap[st]), logger)
 			return
 		}
 		jsonResponse(content, w, logger)
 	case http.MethodDelete:
 		content, err := streamProcessor.DropStream(name, st)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("delete %s error", xsql.StreamTypeMap[st]), logger)
+			handleError(w, err, fmt.Sprintf("delete %s error", ast.StreamTypeMap[st]), logger)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
@@ -212,7 +213,7 @@ func sourceManageHandler(w http.ResponseWriter, r *http.Request, st xsql.StreamT
 		}
 		content, err := streamProcessor.ExecReplaceStream(v.Sql, st)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(xsql.StreamTypeMap[st])), logger)
+			handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
@@ -222,21 +223,21 @@ func sourceManageHandler(w http.ResponseWriter, r *http.Request, st xsql.StreamT
 
 //list or create streams
 func streamsHandler(w http.ResponseWriter, r *http.Request) {
-	sourcesManageHandler(w, r, xsql.TypeStream)
+	sourcesManageHandler(w, r, ast.TypeStream)
 }
 
 //describe or delete a stream
 func streamHandler(w http.ResponseWriter, r *http.Request) {
-	sourceManageHandler(w, r, xsql.TypeStream)
+	sourceManageHandler(w, r, ast.TypeStream)
 }
 
 //list or create tables
 func tablesHandler(w http.ResponseWriter, r *http.Request) {
-	sourcesManageHandler(w, r, xsql.TypeTable)
+	sourcesManageHandler(w, r, ast.TypeTable)
 }
 
 func tableHandler(w http.ResponseWriter, r *http.Request) {
-	sourceManageHandler(w, r, xsql.TypeTable)
+	sourceManageHandler(w, r, ast.TypeTable)
 }
 
 //list or create rules
@@ -406,35 +407,35 @@ func getTopoRuleHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(content))
 }
 
-func pluginsHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType) {
+func pluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	defer r.Body.Close()
 	switch r.Method {
 	case http.MethodGet:
 		content, err := pluginManager.List(t)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s plugins list command error", plugins.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("%s plugins list command error", plugin.PluginTypes[t]), logger)
 			return
 		}
 		jsonResponse(content, w, logger)
 	case http.MethodPost:
-		sd := plugins.NewPluginByType(t)
+		sd := plugin.NewPluginByType(t)
 		err := json.NewDecoder(r.Body).Decode(sd)
 		// Problems decoding
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", plugins.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", plugin.PluginTypes[t]), logger)
 			return
 		}
 		err = pluginManager.Register(t, sd)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s plugins create command error", plugins.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("%s plugins create command error", plugin.PluginTypes[t]), logger)
 			return
 		}
 		w.WriteHeader(http.StatusCreated)
-		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", plugins.PluginTypes[t], sd.GetName())))
+		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", plugin.PluginTypes[t], sd.GetName())))
 	}
 }
 
-func pluginHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType) {
+func pluginHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
@@ -445,11 +446,11 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType)
 		r := cb == "1"
 		err := pluginManager.Delete(t, name, r)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", plugins.PluginTypes[t], name), logger)
+			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", plugin.PluginTypes[t], name), logger)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
-		result := fmt.Sprintf("%s plugin %s is deleted", plugins.PluginTypes[t], name)
+		result := fmt.Sprintf("%s plugin %s is deleted", plugin.PluginTypes[t], name)
 		if r {
 			result = fmt.Sprintf("%s and Kuiper will be stopped", result)
 		} else {
@@ -459,7 +460,7 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType)
 	case http.MethodGet:
 		j, ok := pluginManager.Get(t, name)
 		if !ok {
-			handleError(w, common.NewErrorWithCode(common.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", plugins.PluginTypes[t], name), logger)
+			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", plugin.PluginTypes[t], name), logger)
 			return
 		}
 		jsonResponse(j, w, logger)
@@ -468,27 +469,27 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType)
 
 //list or create source plugin
 func sourcesHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugins.SOURCE)
+	pluginsHandler(w, r, plugin.SOURCE)
 }
 
 //delete a source plugin
 func sourceHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugins.SOURCE)
+	pluginHandler(w, r, plugin.SOURCE)
 }
 
 //list or create sink plugin
 func sinksHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugins.SINK)
+	pluginsHandler(w, r, plugin.SINK)
 }
 
 //delete a sink plugin
 func sinkHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugins.SINK)
+	pluginHandler(w, r, plugin.SINK)
 }
 
 //list or create function plugin
 func functionsHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugins.FUNCTION)
+	pluginsHandler(w, r, plugin.FUNCTION)
 }
 
 //list all user defined functions in all function plugins
@@ -506,7 +507,7 @@ func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 	name := vars["name"]
 	j, ok := pluginManager.GetSymbol(name)
 	if !ok {
-		handleError(w, common.NewErrorWithCode(common.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
+		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
 		return
 	}
 	jsonResponse(map[string]string{"name": name, "plugin": j}, w, logger)
@@ -514,7 +515,7 @@ func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 
 //delete a function plugin
 func functionHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugins.FUNCTION)
+	pluginHandler(w, r, plugin.FUNCTION)
 }
 
 type functionList struct {
@@ -529,9 +530,9 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	name := vars["name"]
 
-	_, ok := pluginManager.Get(plugins.FUNCTION, name)
+	_, ok := pluginManager.Get(plugin.FUNCTION, name)
 	if !ok {
-		handleError(w, common.NewErrorWithCode(common.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", plugins.PluginTypes[plugins.FUNCTION], name), logger)
+		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", plugin.PluginTypes[plugin.FUNCTION], name), logger)
 		return
 	}
 	sd := functionList{}
@@ -551,15 +552,15 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func prebuildSourcePlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugins.SOURCE)
+	prebuildPluginsHandler(w, r, plugin.SOURCE)
 }
 
 func prebuildSinkPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugins.SINK)
+	prebuildPluginsHandler(w, r, plugin.SINK)
 }
 
 func prebuildFuncsPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugins.FUNCTION)
+	prebuildPluginsHandler(w, r, plugin.FUNCTION)
 }
 
 func isOffcialDockerImage() bool {
@@ -569,13 +570,13 @@ func isOffcialDockerImage() bool {
 	return true
 }
 
-func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugins.PluginType) {
+func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	emsg := "It's strongly recommended to install plugins at official released Debian Docker images. If you choose to proceed to install plugin, please make sure the plugin is already validated in your own build."
 	if !isOffcialDockerImage() {
 		handleError(w, fmt.Errorf(emsg), "", logger)
 		return
 	} else if runtime.GOOS == "linux" {
-		osrelease, err := common.Read()
+		osrelease, err := Read()
 		if err != nil {
 			logger.Infof("")
 			return
@@ -583,11 +584,11 @@ func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugins.Pl
 		prettyName := strings.ToUpper(osrelease["PRETTY_NAME"])
 		os := "debian"
 		if strings.Contains(prettyName, "DEBIAN") {
-			hosts := common.Config.Basic.PluginHosts
+			hosts := conf.Config.Basic.PluginHosts
 			ptype := "sources"
-			if t == plugins.SINK {
+			if t == plugin.SINK {
 				ptype = "sinks"
-			} else if t == plugins.FUNCTION {
+			} else if t == plugin.FUNCTION {
 				ptype = "functions"
 			}
 			if err, plugins := fetchPluginList(hosts, ptype, os, runtime.GOARCH); err != nil {
@@ -686,7 +687,7 @@ loop:
 //list sink plugin
 func sinksMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	sinks := plugins.GetSinks()
+	sinks := plugin.GetSinks()
 	jsonResponse(sinks, w, logger)
 	return
 }
@@ -698,7 +699,7 @@ func newSinkMetaHandler(w http.ResponseWriter, r *http.Request) {
 	pluginName := vars["name"]
 
 	language := getLanguage(r)
-	ptrMetadata, err := plugins.GetSinkMeta(pluginName, language)
+	ptrMetadata, err := plugin.GetSinkMeta(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -709,7 +710,7 @@ func newSinkMetaHandler(w http.ResponseWriter, r *http.Request) {
 //list functions
 func functionsMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	sinks := plugins.GetFunctions()
+	sinks := plugin.GetFunctions()
 	jsonResponse(sinks, w, logger)
 	return
 }
@@ -717,7 +718,7 @@ func functionsMetaHandler(w http.ResponseWriter, r *http.Request) {
 //list source plugin
 func sourcesMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	ret := plugins.GetSources()
+	ret := plugin.GetSources()
 	if nil != ret {
 		jsonResponse(ret, w, logger)
 		return
@@ -730,7 +731,7 @@ func sourceMetaHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
 	language := getLanguage(r)
-	ret, err := plugins.GetSourceMeta(pluginName, language)
+	ret, err := plugin.GetSourceMeta(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -747,7 +748,7 @@ func sourceConfHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
 	language := getLanguage(r)
-	ret, err := plugins.GetSourceConf(pluginName, language)
+	ret, err := plugin.GetSourceConf(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -761,7 +762,7 @@ func sourceConfKeysHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
-	ret := plugins.GetSourceConfKeys(pluginName)
+	ret := plugin.GetSourceConfKeys(pluginName)
 	if nil != ret {
 		jsonResponse(ret, w, logger)
 		return
@@ -780,14 +781,14 @@ func sourceConfKeyHandler(w http.ResponseWriter, r *http.Request) {
 	language := getLanguage(r)
 	switch r.Method {
 	case http.MethodDelete:
-		err = plugins.DelSourceConfKey(pluginName, confKey, language)
+		err = plugin.DelSourceConfKey(pluginName, confKey, language)
 	case http.MethodPost:
 		v, err := ioutil.ReadAll(r.Body)
 		if err != nil {
 			handleError(w, err, "Invalid body", logger)
 			return
 		}
-		err = plugins.AddSourceConfKey(pluginName, confKey, language, v)
+		err = plugin.AddSourceConfKey(pluginName, confKey, language, v)
 	}
 	if err != nil {
 		handleError(w, err, "", logger)
@@ -816,9 +817,9 @@ func sourceConfKeyFieldsHandler(w http.ResponseWriter, r *http.Request) {
 	language := getLanguage(r)
 	switch r.Method {
 	case http.MethodDelete:
-		err = plugins.DelSourceConfKeyField(pluginName, confKey, language, v)
+		err = plugin.DelSourceConfKeyField(pluginName, confKey, language, v)
 	case http.MethodPost:
-		err = plugins.AddSourceConfKeyField(pluginName, confKey, language, v)
+		err = plugin.AddSourceConfKeyField(pluginName, confKey, language, v)
 	}
 	if err != nil {
 		handleError(w, err, "", logger)
@@ -848,7 +849,7 @@ func servicesHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		jsonResponse(content, w, logger)
 	case http.MethodPost:
-		sd := &services.ServiceCreationRequest{}
+		sd := &service.ServiceCreationRequest{}
 		err := json.NewDecoder(r.Body).Decode(sd)
 		// Problems decoding
 		if err != nil {
@@ -883,12 +884,12 @@ func serviceHandler(w http.ResponseWriter, r *http.Request) {
 	case http.MethodGet:
 		j, err := serviceManager.Get(name)
 		if err != nil {
-			handleError(w, common.NewErrorWithCode(common.NOT_FOUND, "not found"), fmt.Sprintf("describe service %s error", name), logger)
+			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe service %s error", name), logger)
 			return
 		}
 		jsonResponse(j, w, logger)
 	case http.MethodPut:
-		sd := &services.ServiceCreationRequest{}
+		sd := &service.ServiceCreationRequest{}
 		err := json.NewDecoder(r.Body).Decode(sd)
 		// Problems decoding
 		if err != nil {
@@ -920,7 +921,7 @@ func serviceFunctionHandler(w http.ResponseWriter, r *http.Request) {
 	name := vars["name"]
 	j, err := serviceManager.GetFunction(name)
 	if err != nil {
-		handleError(w, common.NewErrorWithCode(common.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
+		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
 		return
 	}
 	jsonResponse(j, w, logger)

xstream/server/server/rpc.go

@@ -4,28 +4,27 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/plugins"
-	"github.com/emqx/kuiper/services"
-	"github.com/emqx/kuiper/xstream/sinks"
+	"github.com/emqx/kuiper/internal/plugin"
+	"github.com/emqx/kuiper/internal/service"
+	"github.com/emqx/kuiper/internal/topo/sinks"
 	"strings"
 	"time"
 )
 
-const QUERY_RULE_ID = "internal-xstream_query_rule"
+const QueryRuleId = "internal-ekuiper_query_rule"
 
 type Server int
 
 func (t *Server) CreateQuery(sql string, reply *string) error {
-	if _, ok := registry.Load(QUERY_RULE_ID); ok {
+	if _, ok := registry.Load(QueryRuleId); ok {
 		stopQuery()
 	}
-	tp, err := ruleProcessor.ExecQuery(QUERY_RULE_ID, sql)
+	tp, err := ruleProcessor.ExecQuery(QueryRuleId, sql)
 	if err != nil {
 		return err
 	} else {
-		rs := &RuleState{Name: QUERY_RULE_ID, Topology: tp, Triggered: true}
-		registry.Store(QUERY_RULE_ID, rs)
+		rs := &RuleState{Name: QueryRuleId, Topology: tp, Triggered: true}
+		registry.Store(QueryRuleId, rs)
 		msg := fmt.Sprintf("Query was submit successfully.")
 		logger.Println(msg)
 		*reply = fmt.Sprintf(msg)
@@ -34,10 +33,10 @@ func (t *Server) CreateQuery(sql string, reply *string) error {
 }
 
 func stopQuery() {
-	if rs, ok := registry.Load(QUERY_RULE_ID); ok {
+	if rs, ok := registry.Load(QueryRuleId); ok {
 		logger.Printf("stop the query.")
 		(*rs.Topology).Cancel()
-		registry.Delete(QUERY_RULE_ID)
+		registry.Delete(QueryRuleId)
 	}
 }
 
@@ -45,7 +44,7 @@ func stopQuery() {
  * qid is not currently used.
  */
 func (t *Server) GetQueryResult(qid string, reply *string) error {
-	if rs, ok := registry.Load(QUERY_RULE_ID); ok {
+	if rs, ok := registry.Load(QueryRuleId); ok {
 		c := (*rs.Topology).GetContext()
 		if c != nil && c.Err() != nil {
 			return c.Err()
@@ -76,7 +75,7 @@ func (t *Server) Stream(stream string, reply *string) error {
 	return nil
 }
 
-func (t *Server) CreateRule(rule *common.RPCArgDesc, reply *string) error {
+func (t *Server) CreateRule(rule *RPCArgDesc, reply *string) error {
 	r, err := ruleProcessor.ExecCreate(rule.Name, rule.Json)
 	if err != nil {
 		return fmt.Errorf("Create rule error : %s.", err)
@@ -187,8 +186,8 @@ func (t *Server) DropRule(name string, reply *string) error {
 	return nil
 }
 
-func (t *Server) CreatePlugin(arg *common.PluginDesc, reply *string) error {
-	pt := plugins.PluginType(arg.Type)
+func (t *Server) CreatePlugin(arg *PluginDesc, reply *string) error {
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
@@ -205,8 +204,8 @@ func (t *Server) CreatePlugin(arg *common.PluginDesc, reply *string) error {
 	return nil
 }
 
-func (t *Server) RegisterPlugin(arg *common.PluginDesc, reply *string) error {
-	p, err := getPluginByJson(arg, plugins.FUNCTION)
+func (t *Server) RegisterPlugin(arg *PluginDesc, reply *string) error {
+	p, err := getPluginByJson(arg, plugin.FUNCTION)
 	if err != nil {
 		return fmt.Errorf("Register plugin functions error: %s", err)
 	}
@@ -222,8 +221,8 @@ func (t *Server) RegisterPlugin(arg *common.PluginDesc, reply *string) error {
 	return nil
 }
 
-func (t *Server) DropPlugin(arg *common.PluginDesc, reply *string) error {
-	pt := plugins.PluginType(arg.Type)
+func (t *Server) DropPlugin(arg *PluginDesc, reply *string) error {
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Drop plugin error: %s", err)
@@ -243,7 +242,7 @@ func (t *Server) DropPlugin(arg *common.PluginDesc, reply *string) error {
 }
 
 func (t *Server) ShowPlugins(arg int, reply *string) error {
-	pt := plugins.PluginType(arg)
+	pt := plugin.PluginType(arg)
 	l, err := pluginManager.List(pt)
 	if err != nil {
 		return fmt.Errorf("Show plugin error: %s", err)
@@ -269,8 +268,8 @@ func (t *Server) ShowUdfs(_ int, reply *string) error {
 	return nil
 }
 
-func (t *Server) DescPlugin(arg *common.PluginDesc, reply *string) error {
-	pt := plugins.PluginType(arg.Type)
+func (t *Server) DescPlugin(arg *PluginDesc, reply *string) error {
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Describe plugin error: %s", err)
@@ -306,8 +305,8 @@ func (t *Server) DescUdf(arg string, reply *string) error {
 	return nil
 }
 
-func (t *Server) CreateService(arg *common.RPCArgDesc, reply *string) error {
-	sd := &services.ServiceCreationRequest{}
+func (t *Server) CreateService(arg *RPCArgDesc, reply *string) error {
+	sd := &service.ServiceCreationRequest{}
 	if arg.Json != "" {
 		if err := json.Unmarshal([]byte(arg.Json), sd); err != nil {
 			return fmt.Errorf("Parse service %s error : %s.", arg.Json, err)
@@ -411,8 +410,8 @@ func marshalDesc(m interface{}) (string, error) {
 	return dst.String(), nil
 }
 
-func getPluginByJson(arg *common.PluginDesc, pt plugins.PluginType) (plugins.Plugin, error) {
-	p := plugins.NewPluginByType(pt)
+func getPluginByJson(arg *PluginDesc, pt plugin.PluginType) (plugin.Plugin, error) {
+	p := plugin.NewPluginByType(pt)
 	if arg.Json != "" {
 		if err := json.Unmarshal([]byte(arg.Json), p); err != nil {
 			return nil, fmt.Errorf("Parse plugin %s error : %s.", arg.Json, err)
@@ -427,7 +426,7 @@ func init() {
 	go func() {
 		for {
 			<-ticker.C
-			if _, ok := registry.Load(QUERY_RULE_ID); !ok {
+			if _, ok := registry.Load(QueryRuleId); !ok {
 				continue
 			}
 

xstream/server/server/ruleManager.go

@@ -5,26 +5,25 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/xstream/planner"
+	"github.com/emqx/kuiper/internal/topo"
+	"github.com/emqx/kuiper/internal/topo/planner"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/errorx"
 	"sort"
 	"sync"
-
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream"
-	"github.com/emqx/kuiper/xstream/api"
 )
 
 var registry *RuleRegistry
 
 type RuleState struct {
 	Name      string
-	Topology  *xstream.TopologyNew
+	Topology  *topo.Topo
 	Triggered bool
 	// temporary storage for topo graph to make sure even rule close, the graph is still available
-	topoGraph *xstream.PrintableTopo
+	topoGraph *topo.PrintableTopo
 }
 
-func (rs *RuleState) GetTopoGraph() *xstream.PrintableTopo {
+func (rs *RuleState) GetTopoGraph() *topo.PrintableTopo {
 	if rs.topoGraph != nil {
 		return rs.topoGraph
 	} else if rs.Topology != nil {
@@ -187,7 +186,7 @@ func getRuleStatus(name string) (string, error) {
 		}
 		return result, nil
 	} else {
-		return "", common.NewErrorWithCode(common.NOT_FOUND, fmt.Sprintf("Rule %s is not found", name))
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("Rule %s is not found", name))
 	}
 }
 
@@ -195,16 +194,16 @@ func getRuleTopo(name string) (string, error) {
 	if rs, ok := registry.Load(name); ok {
 		topo := rs.GetTopoGraph()
 		if topo == nil {
-			return "", common.NewError(fmt.Sprintf("Fail to get rule %s's topo, make sure the rule has been started before", name))
+			return "", errorx.New(fmt.Sprintf("Fail to get rule %s's topo, make sure the rule has been started before", name))
 		}
 		bytes, err := json.Marshal(topo)
 		if err != nil {
-			return "", common.NewError(fmt.Sprintf("Fail to encode rule %s's topo", name))
+			return "", errorx.New(fmt.Sprintf("Fail to encode rule %s's topo", name))
 		} else {
 			return string(bytes), nil
 		}
 	} else {
-		return "", common.NewErrorWithCode(common.NOT_FOUND, fmt.Sprintf("Rule %s is not found", name))
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("Rule %s is not found", name))
 	}
 }
 

xstream/server/server/server.go

@@ -1,11 +1,11 @@
 package server
 
 import (
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/plugins"
-	"github.com/emqx/kuiper/services"
-	"github.com/emqx/kuiper/xsql"
-	"github.com/emqx/kuiper/xsql/processors"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/plugin"
+	"github.com/emqx/kuiper/internal/processor"
+	"github.com/emqx/kuiper/internal/service"
+	"github.com/emqx/kuiper/internal/xsql"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"context"
@@ -21,22 +21,22 @@ import (
 
 var (
 	dataDir         string
-	logger          = common.Log
+	logger          = conf.Log
 	startTimeStamp  int64
 	version         = ""
-	ruleProcessor   *processors.RuleProcessor
-	streamProcessor *processors.StreamProcessor
-	pluginManager   *plugins.Manager
-	serviceManager  *services.Manager
+	ruleProcessor   *processor.RuleProcessor
+	streamProcessor *processor.StreamProcessor
+	pluginManager   *plugin.Manager
+	serviceManager  *service.Manager
 )
 
 func StartUp(Version, LoadFileType string) {
 	version = Version
-	common.LoadFileType = LoadFileType
+	conf.LoadFileType = LoadFileType
 	startTimeStamp = time.Now().Unix()
-	common.InitConf()
+	conf.InitConf()
 
-	dr, err := common.GetDataLoc()
+	dr, err := conf.GetDataLoc()
 	if err != nil {
 		logger.Panic(err)
 	} else {
@@ -44,13 +44,13 @@ func StartUp(Version, LoadFileType string) {
 		dataDir = dr
 	}
 
-	ruleProcessor = processors.NewRuleProcessor(dataDir)
-	streamProcessor = processors.NewStreamProcessor(path.Join(dataDir, "stream"))
-	pluginManager, err = plugins.NewPluginManager()
+	ruleProcessor = processor.NewRuleProcessor(dataDir)
+	streamProcessor = processor.NewStreamProcessor(path.Join(dataDir, "stream"))
+	pluginManager, err = plugin.NewPluginManager()
 	if err != nil {
 		logger.Panic(err)
 	}
-	serviceManager, err = services.GetServiceManager()
+	serviceManager, err = service.GetServiceManager()
 	if err != nil {
 		logger.Panic(err)
 	}
@@ -76,8 +76,8 @@ func StartUp(Version, LoadFileType string) {
 
 	//Start prometheus service
 	var srvPrometheus *http.Server = nil
-	if common.Config.Basic.Prometheus {
-		portPrometheus := common.Config.Basic.PrometheusPort
+	if conf.Config.Basic.Prometheus {
+		portPrometheus := conf.Config.Basic.PrometheusPort
 		if portPrometheus <= 0 {
 			logger.Fatal("Miss configuration prometheusPort")
 		}
@@ -101,13 +101,13 @@ func StartUp(Version, LoadFileType string) {
 	}
 
 	//Start rest service
-	srvRest := createRestServer(common.Config.Basic.RestIp, common.Config.Basic.RestPort)
+	srvRest := createRestServer(conf.Config.Basic.RestIp, conf.Config.Basic.RestPort)
 	go func() {
 		var err error
-		if common.Config.Basic.RestTls == nil {
+		if conf.Config.Basic.RestTls == nil {
 			err = srvRest.ListenAndServe()
 		} else {
-			err = srvRest.ListenAndServeTLS(common.Config.Basic.RestTls.Certfile, common.Config.Basic.RestTls.Keyfile)
+			err = srvRest.ListenAndServeTLS(conf.Config.Basic.RestTls.Certfile, conf.Config.Basic.RestTls.Keyfile)
 		}
 		if err != nil && err != http.ErrServerClosed {
 			logger.Fatal("Error serving rest service: ", err)
@@ -115,8 +115,8 @@ func StartUp(Version, LoadFileType string) {
 	}()
 
 	// Start rpc service
-	portRpc := common.Config.Basic.Port
-	ipRpc := common.Config.Basic.Ip
+	portRpc := conf.Config.Basic.Port
+	ipRpc := conf.Config.Basic.Ip
 	rpcSrv := rpc.NewServer()
 	err = rpcSrv.Register(server)
 	if err != nil {
@@ -137,10 +137,10 @@ func StartUp(Version, LoadFileType string) {
 
 	//Startup message
 	restHttpType := "http"
-	if common.Config.Basic.RestTls != nil {
+	if conf.Config.Basic.RestTls != nil {
 		restHttpType = "https"
 	}
-	msg := fmt.Sprintf("Serving kuiper (version - %s) on port %d, and restful api on %s://%s:%d. \n", Version, common.Config.Basic.Port, restHttpType, common.Config.Basic.RestIp, common.Config.Basic.RestPort)
+	msg := fmt.Sprintf("Serving kuiper (version - %s) on port %d, and restful api on %s://%s:%d. \n", Version, conf.Config.Basic.Port, restHttpType, conf.Config.Basic.RestIp, conf.Config.Basic.RestPort)
 	logger.Info(msg)
 	fmt.Printf(msg)
 

services/executors.go

@@ -1,11 +1,12 @@
-package services
+package service
 
 import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/pkg/httpx"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
 	"github.com/golang/protobuf/proto"
 	"github.com/jhump/protoreflect/dynamic"
 	"github.com/jhump/protoreflect/dynamic/grpcdynamic"
@@ -55,7 +56,7 @@ func NewExecutor(i *interfaceInfo) (executor, error) {
 			return nil, fmt.Errorf("invalid descriptor type for rest")
 		}
 		o := &restOption{}
-		e := common.MapToStruct(i.Options, o)
+		e := cast.MapToStruct(i.Options, o)
 		if e != nil {
 			return nil, fmt.Errorf("incorrect rest option: %v", e)
 		}
@@ -191,7 +192,7 @@ func (h *httpExecutor) InvokeFunction(ctx api.FunctionContext, name string, para
 	if err != nil {
 		return nil, err
 	}
-	resp, err := common.Send(ctx.GetLogger(), h.conn, "json", hm.Method, u, h.restOpt.Headers, false, hm.Body)
+	resp, err := httpx.Send(ctx.GetLogger(), h.conn, "json", hm.Method, u, h.restOpt.Headers, false, hm.Body)
 	if err != nil {
 		return nil, err
 	}

services/external_func.go

@@ -1,6 +1,8 @@
-package services
+package service
 
-import "github.com/emqx/kuiper/xstream/api"
+import (
+	"github.com/emqx/kuiper/pkg/api"
+)
 
 type ExternalFunc struct {
 	methodName string

services/external_service_rule_test.go

@@ -1,13 +1,13 @@
-package services
+package service
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	pb "github.com/emqx/kuiper/services/test/schemas/helloworld"
-	"github.com/emqx/kuiper/xstream/api"
-	"github.com/emqx/kuiper/xstream/topotest"
+	kconf "github.com/emqx/kuiper/internal/conf"
+	pb "github.com/emqx/kuiper/internal/service/test/schemas/helloworld"
+	"github.com/emqx/kuiper/internal/topo/topotest"
+	"github.com/emqx/kuiper/pkg/api"
 	"github.com/golang/protobuf/ptypes/empty"
 	"github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/gorilla/mux"
@@ -746,13 +746,13 @@ func (s *server) GetStatus(context.Context, *empty.Empty) (*wrappers.BoolValue,
 func TestGrpcService(t *testing.T) {
 	lis, err := net.Listen("tcp", ":50051")
 	if err != nil {
-		common.Log.Fatalf("failed to listen: %v", err)
+		kconf.Log.Fatalf("failed to listen: %v", err)
 	}
 	s := grpc.NewServer()
 	pb.RegisterGreeterServer(s, &server{})
 	go func() {
 		if err := s.Serve(lis); err != nil {
-			common.Log.Fatalf("failed to serve: %v", err)
+			kconf.Log.Fatalf("failed to serve: %v", err)
 		}
 	}()
 	defer s.Stop()

services/manager.go

@@ -1,11 +1,13 @@
-package services
+package service
 
 import (
 	"archive/zip"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/common/kv"
-	"github.com/emqx/kuiper/xstream/api"
+	kconf "github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
+	"github.com/emqx/kuiper/internal/pkg/httpx"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/kv"
 	"io/ioutil"
 	"os"
 	"path"
@@ -36,14 +38,14 @@ func GetServiceManager() (*Manager, error) {
 	defer mutex.Unlock()
 	if singleton == nil {
 		dir := "etc/services"
-		if common.IsTesting {
-			dir = "services/test"
+		if kconf.IsTesting {
+			dir = "service/test"
 		}
-		etcDir, err := common.GetLoc(dir)
+		etcDir, err := kconf.GetLoc(dir)
 		if err != nil {
 			return nil, fmt.Errorf("cannot find etc/services folder: %s", err)
 		}
-		dbDir, err := common.GetDataLoc()
+		dbDir, err := kconf.GetDataLoc()
 		if err != nil {
 			return nil, fmt.Errorf("cannot find db folder: %s", err)
 		}
@@ -67,7 +69,7 @@ func GetServiceManager() (*Manager, error) {
 			functionKV: fdb,
 		}
 	}
-	if !singleton.loaded && !common.IsTesting { // To boost the testing perf
+	if !singleton.loaded && !kconf.IsTesting { // To boost the testing perf
 		err := singleton.InitByFiles()
 		return singleton, err
 	}
@@ -83,7 +85,7 @@ func GetServiceManager() (*Manager, error) {
  * NOT threadsafe, must run in lock
  */
 func (m *Manager) InitByFiles() error {
-	common.Log.Debugf("init service manager")
+	kconf.Log.Debugf("init service manager")
 	files, err := ioutil.ReadDir(m.etcDir)
 	if nil != err {
 		return err
@@ -94,7 +96,7 @@ func (m *Manager) InitByFiles() error {
 		if filepath.Ext(baseName) == ".json" {
 			err := m.initFile(baseName)
 			if err != nil {
-				common.Log.Errorf("%v", err)
+				kconf.Log.Errorf("%v", err)
 				continue
 			}
 		}
@@ -105,7 +107,7 @@ func (m *Manager) InitByFiles() error {
 
 func (m *Manager) initFile(baseName string) error {
 	serviceConf := &conf{}
-	err := common.ReadJsonUnmarshal(filepath.Join(m.etcDir, baseName), serviceConf)
+	err := filex.ReadJsonUnmarshal(filepath.Join(m.etcDir, baseName), serviceConf)
 	if err != nil {
 		return fmt.Errorf("parse services file %s failed: %v", baseName, err)
 	}
@@ -154,7 +156,7 @@ func (m *Manager) initFile(baseName string) error {
 				MethodName:    methods[i],
 			})
 			if err != nil {
-				common.Log.Errorf("fail to save the function mapping for %s, the function is not available: %v", f, err)
+				kconf.Log.Errorf("fail to save the function mapping for %s, the function is not available: %v", f, err)
 			}
 		}
 	}
@@ -169,7 +171,7 @@ func (m *Manager) initFile(baseName string) error {
 
 func (m *Manager) HasFunction(name string) bool {
 	_, ok := m.getFunction(name)
-	common.Log.Debugf("found external function %s? %v ", name, ok)
+	kconf.Log.Debugf("found external function %s? %v ", name, ok)
 	return ok
 }
 
@@ -198,7 +200,7 @@ func (m *Manager) Function(name string) (api.Function, error) {
 
 func (m *Manager) HasService(name string) bool {
 	_, ok := m.getService(name)
-	common.Log.Debugf("found external service %s? %v ", name, ok)
+	kconf.Log.Debugf("found external service %s? %v ", name, ok)
 	return ok
 }
 
@@ -211,7 +213,7 @@ func (m *Manager) getFunction(name string) (*functionContainer, bool) {
 		r = &functionContainer{}
 		ok, err := m.functionKV.Get(name, r)
 		if err != nil {
-			common.Log.Errorf("failed to get service function %s from kv: %v", name, err)
+			kconf.Log.Errorf("failed to get service function %s from kv: %v", name, err)
 			return nil, false
 		}
 		if ok {
@@ -230,7 +232,7 @@ func (m *Manager) getService(name string) (*serviceInfo, bool) {
 		r = &serviceInfo{}
 		ok, err := m.serviceKV.Get(name, r)
 		if err != nil {
-			common.Log.Errorf("failed to get service %s from kv: %v", name, err)
+			kconf.Log.Errorf("failed to get service %s from kv: %v", name, err)
 			return nil, false
 		}
 		if ok {
@@ -292,14 +294,14 @@ func (m *Manager) Create(r *ServiceCreationRequest) error {
 	if ok, _ := m.serviceKV.Get(name, &serviceInfo{}); ok {
 		return fmt.Errorf("service %s exist", name)
 	}
-	if !common.IsValidUrl(uri) || !strings.HasSuffix(uri, ".zip") {
+	if !httpx.IsValidUrl(uri) || !strings.HasSuffix(uri, ".zip") {
 		return fmt.Errorf("invalid file path %s", uri)
 	}
 	zipPath := path.Join(m.etcDir, name+".zip")
 	//clean up: delete zip file and unzip files in error
 	defer os.Remove(zipPath)
 	//download
-	err := common.DownloadFile(zipPath, uri)
+	err := httpx.DownloadFile(zipPath, uri)
 	if err != nil {
 		return fmt.Errorf("fail to download file %s: %s", uri, err)
 	}
@@ -326,7 +328,7 @@ func (m *Manager) Delete(name string) error {
 	path := path.Join(m.etcDir, name+".json")
 	err = os.Remove(path)
 	if err != nil {
-		common.Log.Errorf("remove service json fails: %v", err)
+		kconf.Log.Errorf("remove service json fails: %v", err)
 	}
 	return nil
 }
@@ -371,7 +373,7 @@ func (m *Manager) unzip(name, src string) error {
 	}
 	// unzip
 	for _, file := range r.File {
-		err := common.UnzipTo(file, path.Join(m.etcDir, file.Name))
+		err := filex.UnzipTo(file, path.Join(m.etcDir, file.Name))
 		if err != nil {
 			return err
 		}

services/manager_test.go

@@ -1,7 +1,7 @@
-package services
+package service
 
 import (
-	"github.com/emqx/kuiper/xsql"
+	"github.com/emqx/kuiper/internal/xsql"
 	"reflect"
 	"testing"
 )

services/model.go

@@ -1,4 +1,4 @@
-package services
+package service
 
 type (
 	protocol string

services/schema.go

@@ -1,10 +1,11 @@
-package services
+package service
 
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xsql"
+	kconf "github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/xsql"
+	"github.com/emqx/kuiper/pkg/cast"
 	"github.com/golang/protobuf/proto"
 	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
 	"github.com/jhump/protoreflect/desc"
@@ -81,10 +82,10 @@ var ( //Do not call these directly, use the get methods
 func ProtoParser() *protoparse.Parser {
 	once.Do(func() {
 		dir := "etc/services/schemas/"
-		if common.IsTesting {
-			dir = "services/test/schemas/"
+		if kconf.IsTesting {
+			dir = "service/test/schemas/"
 		}
-		schemaDir, _ := common.GetLoc(dir)
+		schemaDir, _ := kconf.GetLoc(dir)
 		protoParser = &protoparse.Parser{ImportPaths: []string{schemaDir}}
 	})
 	return protoParser
@@ -178,7 +179,7 @@ func (d *wrappedProtoDescriptor) ConvertParamsToJson(method string, params []int
 		}
 		im := m.GetInputType()
 		if im.GetFullyQualifiedName() == wrapperString {
-			ss, err := common.ToString(params[0], common.STRICT)
+			ss, err := cast.ToString(params[0], cast.STRICT)
 			if err != nil {
 				return nil, err
 			}
@@ -215,7 +216,7 @@ func (d *wrappedProtoDescriptor) convertParams(im *desc.MessageDescriptor, param
 		// If it is map, try unfold it
 		// TODO custom error for non map or map name not match
 		if r, err := d.unfoldMap(im, params[0]); err != nil {
-			common.Log.Debugf("try unfold param for message %s fail: %v", im.GetName(), err)
+			kconf.Log.Debugf("try unfold param for message %s fail: %v", im.GetName(), err)
 		} else {
 			return r, nil
 		}
@@ -249,10 +250,10 @@ func (d *wrappedProtoDescriptor) ConvertReturn(method string, returnVal interfac
 	m := d.MethodDescriptor(method)
 	t := m.GetOutputType()
 	if _, ok := WRAPPER_TYPES[t.GetFullyQualifiedName()]; ok {
-		return decodeField(returnVal, t.FindFieldByNumber(1), common.STRICT)
+		return decodeField(returnVal, t.FindFieldByNumber(1), cast.STRICT)
 	} else { // MUST be a map
 		if retMap, ok := returnVal.(map[string]interface{}); ok {
-			return decodeMap(retMap, t, common.CONVERT_SAMEKIND)
+			return decodeMap(retMap, t, cast.CONVERT_SAMEKIND)
 		} else {
 			return nil, fmt.Errorf("fail to convert return val, must be a map but got %v", returnVal)
 		}
@@ -271,14 +272,14 @@ func (d *wrappedProtoDescriptor) ConvertReturnJson(method string, returnVal []by
 		return nil, err
 	}
 	m := d.MethodDescriptor(method)
-	return decodeMap(r, m.GetOutputType(), common.CONVERT_SAMEKIND)
+	return decodeMap(r, m.GetOutputType(), cast.CONVERT_SAMEKIND)
 }
 
 func (d *wrappedProtoDescriptor) ConvertReturnText(method string, returnVal []byte) (interface{}, error) {
 	m := d.MethodDescriptor(method)
 	t := m.GetOutputType()
 	if _, ok := WRAPPER_TYPES[t.GetFullyQualifiedName()]; ok {
-		return decodeField(string(returnVal), t.FindFieldByNumber(1), common.CONVERT_ALL)
+		return decodeField(string(returnVal), t.FindFieldByNumber(1), cast.CONVERT_ALL)
 	} else {
 		return nil, fmt.Errorf("fail to convert return val to text, return type must be primitive type but got %s", t.GetName())
 	}
@@ -345,53 +346,53 @@ func (d *wrappedProtoDescriptor) encodeField(field *desc.FieldDescriptor, v inte
 		)
 		switch ft {
 		case dpb.FieldDescriptorProto_TYPE_DOUBLE:
-			result, err = common.ToFloat64Slice(v, common.STRICT)
+			result, err = cast.ToFloat64Slice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_FLOAT:
-			result, err = common.ToTypedSlice(v, func(input interface{}, sn common.Strictness) (interface{}, error) {
-				r, err := common.ToFloat64(input, sn)
+			result, err = cast.ToTypedSlice(v, func(input interface{}, sn cast.Strictness) (interface{}, error) {
+				r, err := cast.ToFloat64(input, sn)
 				if err != nil {
 					return 0, nil
 				} else {
 					return float32(r), nil
 				}
-			}, "float", common.STRICT)
+			}, "float", cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_INT32, dpb.FieldDescriptorProto_TYPE_SFIXED32, dpb.FieldDescriptorProto_TYPE_SINT32:
-			result, err = common.ToTypedSlice(v, func(input interface{}, sn common.Strictness) (interface{}, error) {
-				r, err := common.ToInt(input, sn)
+			result, err = cast.ToTypedSlice(v, func(input interface{}, sn cast.Strictness) (interface{}, error) {
+				r, err := cast.ToInt(input, sn)
 				if err != nil {
 					return 0, nil
 				} else {
 					return int32(r), nil
 				}
-			}, "int", common.STRICT)
+			}, "int", cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_INT64, dpb.FieldDescriptorProto_TYPE_SFIXED64, dpb.FieldDescriptorProto_TYPE_SINT64:
-			result, err = common.ToInt64Slice(v, common.STRICT)
+			result, err = cast.ToInt64Slice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_FIXED32, dpb.FieldDescriptorProto_TYPE_UINT32:
-			result, err = common.ToTypedSlice(v, func(input interface{}, sn common.Strictness) (interface{}, error) {
-				r, err := common.ToUint64(input, sn)
+			result, err = cast.ToTypedSlice(v, func(input interface{}, sn cast.Strictness) (interface{}, error) {
+				r, err := cast.ToUint64(input, sn)
 				if err != nil {
 					return 0, nil
 				} else {
 					return uint32(r), nil
 				}
-			}, "uint", common.STRICT)
+			}, "uint", cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_FIXED64, dpb.FieldDescriptorProto_TYPE_UINT64:
-			result, err = common.ToUint64Slice(v, common.STRICT)
+			result, err = cast.ToUint64Slice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_BOOL:
-			result, err = common.ToBoolSlice(v, common.STRICT)
+			result, err = cast.ToBoolSlice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_STRING:
-			result, err = common.ToStringSlice(v, common.STRICT)
+			result, err = cast.ToStringSlice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_BYTES:
-			result, err = common.ToBytesSlice(v, common.STRICT)
+			result, err = cast.ToBytesSlice(v, cast.STRICT)
 		case dpb.FieldDescriptorProto_TYPE_MESSAGE:
-			result, err = common.ToTypedSlice(v, func(input interface{}, sn common.Strictness) (interface{}, error) {
-				r, err := common.ToStringMap(v)
+			result, err = cast.ToTypedSlice(v, func(input interface{}, sn cast.Strictness) (interface{}, error) {
+				r, err := cast.ToStringMap(v)
 				if err == nil {
 					return d.encodeMap(field.GetMessageType(), r)
 				} else {
 					return nil, fmt.Errorf("invalid type for map type field '%s': %v", fn, err)
 				}
-			}, "map", common.STRICT)
+			}, "map", cast.STRICT)
 		default:
 			return nil, fmt.Errorf("invalid type for field '%s'", fn)
 		}
@@ -408,70 +409,70 @@ func (d *wrappedProtoDescriptor) encodeSingleField(field *desc.FieldDescriptor,
 	fn := field.GetName()
 	switch field.GetType() {
 	case dpb.FieldDescriptorProto_TYPE_DOUBLE:
-		r, err := common.ToFloat64(v, common.STRICT)
+		r, err := cast.ToFloat64(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for float type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_FLOAT:
-		r, err := common.ToFloat64(v, common.STRICT)
+		r, err := cast.ToFloat64(v, cast.STRICT)
 		if err == nil {
 			return float32(r), nil
 		} else {
 			return nil, fmt.Errorf("invalid type for float type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_INT32, dpb.FieldDescriptorProto_TYPE_SFIXED32, dpb.FieldDescriptorProto_TYPE_SINT32:
-		r, err := common.ToInt(v, common.STRICT)
+		r, err := cast.ToInt(v, cast.STRICT)
 		if err == nil {
 			return int32(r), nil
 		} else {
 			return nil, fmt.Errorf("invalid type for int type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_INT64, dpb.FieldDescriptorProto_TYPE_SFIXED64, dpb.FieldDescriptorProto_TYPE_SINT64:
-		r, err := common.ToInt64(v, common.STRICT)
+		r, err := cast.ToInt64(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for int type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_FIXED32, dpb.FieldDescriptorProto_TYPE_UINT32:
-		r, err := common.ToUint64(v, common.STRICT)
+		r, err := cast.ToUint64(v, cast.STRICT)
 		if err == nil {
 			return uint32(r), nil
 		} else {
 			return nil, fmt.Errorf("invalid type for uint type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_FIXED64, dpb.FieldDescriptorProto_TYPE_UINT64:
-		r, err := common.ToUint64(v, common.STRICT)
+		r, err := cast.ToUint64(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for uint type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_BOOL:
-		r, err := common.ToBool(v, common.STRICT)
+		r, err := cast.ToBool(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for bool type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_STRING:
-		r, err := common.ToString(v, common.STRICT)
+		r, err := cast.ToString(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for string type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_BYTES:
-		r, err := common.ToBytes(v, common.STRICT)
+		r, err := cast.ToBytes(v, cast.STRICT)
 		if err == nil {
 			return r, nil
 		} else {
 			return nil, fmt.Errorf("invalid type for bytes type field '%s': %v", fn, err)
 		}
 	case dpb.FieldDescriptorProto_TYPE_MESSAGE:
-		r, err := common.ToStringMap(v)
+		r, err := cast.ToStringMap(v)
 		if err == nil {
 			return d.encodeMap(field.GetMessageType(), r)
 		} else {
@@ -490,12 +491,12 @@ func decodeMessage(message *dynamic.Message, outputType *desc.MessageDescriptor)
 	}
 	result := make(map[string]interface{})
 	for _, field := range outputType.GetFields() {
-		decodeMessageField(message.GetField(field), field, result, common.STRICT)
+		decodeMessageField(message.GetField(field), field, result, cast.STRICT)
 	}
 	return result
 }
 
-func decodeMessageField(src interface{}, field *desc.FieldDescriptor, result map[string]interface{}, sn common.Strictness) error {
+func decodeMessageField(src interface{}, field *desc.FieldDescriptor, result map[string]interface{}, sn cast.Strictness) error {
 	if f, err := decodeField(src, field, sn); err != nil {
 		return err
 	} else {
@@ -504,7 +505,7 @@ func decodeMessageField(src interface{}, field *desc.FieldDescriptor, result map
 	}
 }
 
-func decodeField(src interface{}, field *desc.FieldDescriptor, sn common.Strictness) (interface{}, error) {
+func decodeField(src interface{}, field *desc.FieldDescriptor, sn cast.Strictness) (interface{}, error) {
 	var (
 		r interface{}
 		e error
@@ -513,37 +514,37 @@ func decodeField(src interface{}, field *desc.FieldDescriptor, sn common.Strictn
 	switch field.GetType() {
 	case dpb.FieldDescriptorProto_TYPE_DOUBLE, dpb.FieldDescriptorProto_TYPE_FLOAT:
 		if field.IsRepeated() {
-			r, e = common.ToFloat64Slice(src, sn)
+			r, e = cast.ToFloat64Slice(src, sn)
 		} else {
-			r, e = common.ToFloat64(src, sn)
+			r, e = cast.ToFloat64(src, sn)
 		}
 	case dpb.FieldDescriptorProto_TYPE_INT32, dpb.FieldDescriptorProto_TYPE_SFIXED32, dpb.FieldDescriptorProto_TYPE_SINT32, dpb.FieldDescriptorProto_TYPE_INT64, dpb.FieldDescriptorProto_TYPE_SFIXED64, dpb.FieldDescriptorProto_TYPE_SINT64, dpb.FieldDescriptorProto_TYPE_FIXED32, dpb.FieldDescriptorProto_TYPE_UINT32, dpb.FieldDescriptorProto_TYPE_FIXED64, dpb.FieldDescriptorProto_TYPE_UINT64:
 		if field.IsRepeated() {
-			r, e = common.ToInt64Slice(src, sn)
+			r, e = cast.ToInt64Slice(src, sn)
 		} else {
-			r, e = common.ToInt64(src, sn)
+			r, e = cast.ToInt64(src, sn)
 		}
 	case dpb.FieldDescriptorProto_TYPE_BOOL:
 		if field.IsRepeated() {
-			r, e = common.ToBoolSlice(src, sn)
+			r, e = cast.ToBoolSlice(src, sn)
 		} else {
-			r, e = common.ToBool(src, sn)
+			r, e = cast.ToBool(src, sn)
 		}
 	case dpb.FieldDescriptorProto_TYPE_STRING:
 		if field.IsRepeated() {
-			r, e = common.ToStringSlice(src, sn)
+			r, e = cast.ToStringSlice(src, sn)
 		} else {
-			r, e = common.ToString(src, sn)
+			r, e = cast.ToString(src, sn)
 		}
 	case dpb.FieldDescriptorProto_TYPE_BYTES:
 		if field.IsRepeated() {
-			r, e = common.ToBytesSlice(src, sn)
+			r, e = cast.ToBytesSlice(src, sn)
 		} else {
-			r, e = common.ToBytes(src, sn)
+			r, e = cast.ToBytes(src, sn)
 		}
 	case dpb.FieldDescriptorProto_TYPE_MESSAGE:
 		if field.IsRepeated() {
-			r, e = common.ToTypedSlice(src, func(input interface{}, ssn common.Strictness) (interface{}, error) {
+			r, e = cast.ToTypedSlice(src, func(input interface{}, ssn cast.Strictness) (interface{}, error) {
 				return decodeSubMessage(input, field.GetMessageType(), ssn)
 			}, "map", sn)
 		} else {
@@ -558,7 +559,7 @@ func decodeField(src interface{}, field *desc.FieldDescriptor, sn common.Strictn
 	return r, e
 }
 
-func decodeMap(src map[string]interface{}, ft *desc.MessageDescriptor, sn common.Strictness) (map[string]interface{}, error) {
+func decodeMap(src map[string]interface{}, ft *desc.MessageDescriptor, sn cast.Strictness) (map[string]interface{}, error) {
 	result := make(map[string]interface{})
 	for _, field := range ft.GetFields() {
 		val, ok := src[field.GetName()]
@@ -573,12 +574,12 @@ func decodeMap(src map[string]interface{}, ft *desc.MessageDescriptor, sn common
 	return result, nil
 }
 
-func decodeSubMessage(input interface{}, ft *desc.MessageDescriptor, sn common.Strictness) (interface{}, error) {
+func decodeSubMessage(input interface{}, ft *desc.MessageDescriptor, sn cast.Strictness) (interface{}, error) {
 	var m = map[string]interface{}{}
 	switch v := input.(type) {
 	case map[interface{}]interface{}:
 		for k, val := range v {
-			m[common.ToStringAlways(k)] = val
+			m[cast.ToStringAlways(k)] = val
 		}
 		return decodeMap(m, ft, sn)
 	case map[string]interface{}:

services/schema_http.go

@@ -1,8 +1,8 @@
-package services
+package service
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/pkg/cast"
 	dpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
 	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/dynamic"
@@ -135,7 +135,7 @@ func (d *wrappedProtoDescriptor) ConvertHttpMapping(method string, params []inte
 				if err != nil {
 					return nil, err
 				}
-				args[i], err = common.ToString(fv, common.CONVERT_ALL)
+				args[i], err = cast.ToString(fv, cast.CONVERT_ALL)
 				if err != nil {
 					return nil, fmt.Errorf("invalid field %s(%v) as http option, must be string", v.name, fv)
 				}

services/schema_http_test.go

@@ -1,7 +1,7 @@
-package services
+package service
 
 import (
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/testx"
 	"net/http"
 	"reflect"
 	"testing"
@@ -55,7 +55,7 @@ func TestBookstoreConvertHttpMapping(t *testing.T) {
 	}
 	for i, tt := range tests {
 		r, err := d.(httpMapping).ConvertHttpMapping(tt.method, tt.params)
-		if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d : interface error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
 		} else if tt.err == "" && !reflect.DeepEqual(tt.result, r) {
 			t.Errorf("%d \n\ninterface result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.result, r)
@@ -131,7 +131,7 @@ func TestMessagingConvertHttpMapping(t *testing.T) {
 	}
 	for i, tt := range tests {
 		r, err := d.(httpMapping).ConvertHttpMapping(tt.method, tt.params)
-		if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d : interface error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
 		} else if tt.err == "" && !reflect.DeepEqual(tt.result, r) {
 			t.Errorf("%d \n\ninterface result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.result, r)

services/schema_test.go

@@ -1,7 +1,7 @@
-package services
+package service
 
 import (
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/testx"
 	"reflect"
 	"testing"
 )
@@ -111,13 +111,13 @@ func TestConvertParams(t *testing.T) {
 	for i, descriptor := range descriptors {
 		for j, tt := range tests {
 			r, err := descriptor.(interfaceDescriptor).ConvertParams(tt.method, tt.params)
-			if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+			if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 				t.Errorf("%d.%d : interface error mismatch:\n  exp=%s\n  got=%s\n\n", i, j, tt.err, err)
 			} else if tt.err == "" && !reflect.DeepEqual(tt.iresult, r) {
 				t.Errorf("%d.%d \n\ninterface result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, j, tt.iresult, r)
 			}
 			rj, err := descriptor.(jsonDescriptor).ConvertParamsToJson(tt.method, tt.params)
-			if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+			if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 				t.Errorf("%d.%d : json error mismatch:\n  exp=%s\n  got=%s\n\n", i, j, tt.err, err)
 			} else if tt.err == "" && !reflect.DeepEqual(tt.jresult, rj) {
 				t.Errorf("%d.%d \n\njson result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, j, tt.jresult, rj)
@@ -227,13 +227,13 @@ func TestConvertReturns(t *testing.T) {
 	for i, descriptor := range descriptors {
 		for j, tt := range tests {
 			r, err := descriptor.(interfaceDescriptor).ConvertReturn(tt.method, tt.ireturn)
-			if !reflect.DeepEqual(tt.ierr, common.Errstring(err)) {
+			if !reflect.DeepEqual(tt.ierr, testx.Errstring(err)) {
 				t.Errorf("%d.%d : interface error mismatch:\n  exp=%s\n  got=%s\n\n", i, j, tt.ierr, err)
 			} else if tt.ierr == "" && !reflect.DeepEqual(tt.iresult, r) {
 				t.Errorf("%d.%d \n\ninterface result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, j, tt.iresult, r)
 			}
 			rj, err := descriptor.(jsonDescriptor).ConvertReturnJson(tt.method, tt.jreturn)
-			if !reflect.DeepEqual(tt.jerr, common.Errstring(err)) {
+			if !reflect.DeepEqual(tt.jerr, testx.Errstring(err)) {
 				t.Errorf("%d.%d : json error mismatch:\n  exp=%s\n  got=%s\n\n", i, j, tt.jerr, err)
 			} else if tt.jerr == "" && !reflect.DeepEqual(tt.jresult, rj) {
 				t.Errorf("%d.%d \n\njson result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, j, tt.jresult, rj)

common/templates/funcs.go

@@ -1,4 +1,4 @@
-package templates
+package template
 
 import (
 	"encoding/base64"

common/templates/funcs_test.go

@@ -1,9 +1,9 @@
-package templates
+package template
 
 import (
 	"encoding/base64"
 	"fmt"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/testx"
 	"reflect"
 	"testing"
 )
@@ -49,7 +49,7 @@ func TestBase64Encode(t *testing.T) {
 	for i, tt := range tests {
 		result, err := Base64Encode(tt.para)
 		r, _ := base64.StdEncoding.DecodeString(result)
-		if !reflect.DeepEqual(tt.err, common.Errstring(err)) {
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.para, tt.err, err)
 
 		} else if tt.err == "" && !reflect.DeepEqual(tt.expect, string(r)) {

common/test_util.go

@@ -1,4 +1,8 @@
-package common
+package testx
+
+import (
+	"github.com/emqx/kuiper/internal/conf"
+)
 
 // errstring returns the string representation of an error.
 func Errstring(err error) string {
@@ -9,10 +13,10 @@ func Errstring(err error) string {
 }
 
 func GetDbDir() string {
-	InitConf()
-	dbDir, err := GetDataLoc()
+	conf.InitConf()
+	dbDir, err := conf.GetDataLoc()
 	if err != nil {
-		Log.Fatal(err)
+		conf.Log.Fatal(err)
 	}
 	return dbDir
 }

xstream/checkpoints/barrier_handler.go

@@ -1,6 +1,8 @@
 package checkpoints
 
-import "github.com/emqx/kuiper/xstream/api"
+import (
+	"github.com/emqx/kuiper/pkg/api"
+)
 
 type BarrierHandler interface {
 	Process(data *BufferOrEvent, ctx api.StreamContext) bool //If data is barrier return true, else return false

xstream/checkpoints/coordinator.go

@@ -2,8 +2,9 @@ package checkpoints
 
 import (
 	"github.com/benbjohnson/clock"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
 	"sync"
 )
 
@@ -143,11 +144,11 @@ func createBarrierHandler(re Responder, inputCount int, qos api.Qos) BarrierHand
 
 func (c *Coordinator) Activate() error {
 	logger := c.ctx.GetLogger()
-	logger.Infof("Start checkpoint coordinator for rule %s at %d", c.ruleId, common.GetNowInMilli())
+	logger.Infof("Start checkpoint coordinator for rule %s at %d", c.ruleId, conf.GetNowInMilli())
 	if c.ticker != nil {
 		c.ticker.Stop()
 	}
-	c.ticker = common.GetTicker(c.baseInterval)
+	c.ticker = conf.GetTicker(c.baseInterval)
 	tc := c.ticker.C
 	go func() {
 		c.activated = true
@@ -160,7 +161,7 @@ func (c *Coordinator) Activate() error {
 				// TODO Check if all tasks are running
 
 				//Create a pending checkpoint
-				checkpointId := common.TimeToUnixMilli(n)
+				checkpointId := cast.TimeToUnixMilli(n)
 				checkpoint := newPendingCheckpoint(checkpointId, c.tasksToWaitFor)
 				logger.Debugf("Create checkpoint %d", checkpointId)
 				c.pendingCheckpoints.Store(checkpointId, checkpoint)
@@ -171,7 +172,7 @@ func (c *Coordinator) Activate() error {
 							logger.Infof("Fail to trigger checkpoint for source %s with error %v, cancel it", t.GetName(), err)
 							c.cancel(checkpointId)
 						} else {
-							timeout := common.GetTicker(c.timeout)
+							timeout := conf.GetTicker(c.timeout)
 							select {
 							case <-timeout.C:
 								logger.Debugf("Try to cancel checkpoint %d for timeout", checkpointId)

xstream/checkpoints/defs.go

@@ -1,7 +1,7 @@
 package checkpoints
 
 import (
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 )
 
 type StreamTask interface {

xstream/collectors/func.go

@@ -2,7 +2,7 @@ package collectors
 
 import (
 	"errors"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 )
 
 // CollectorFunc is a function used to colllect

xstream/contexts/default.go

@@ -3,8 +3,9 @@ package contexts
 import (
 	"context"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
 	"github.com/sirupsen/logrus"
 	"sync"
 	"time"
@@ -66,7 +67,7 @@ func (c *DefaultContext) GetLogger() api.Logger {
 	if l != nil && ok {
 		return l
 	}
-	return common.Log.WithField("caller", "default")
+	return conf.Log.WithField("caller", "default")
 }
 
 func (c *DefaultContext) GetRuleId() string {
@@ -82,7 +83,7 @@ func (c *DefaultContext) GetInstanceId() int {
 }
 
 func (c *DefaultContext) GetRootPath() string {
-	loc, _ := common.GetLoc("")
+	loc, _ := conf.GetLoc("")
 	return loc
 }
 
@@ -128,7 +129,7 @@ func (c *DefaultContext) WithCancel() (api.StreamContext, context.CancelFunc) {
 
 func (c *DefaultContext) IncrCounter(key string, amount int) error {
 	if v, ok := c.state.Load(key); ok {
-		if vi, err := common.ToInt(v, common.STRICT); err != nil {
+		if vi, err := cast.ToInt(v, cast.STRICT); err != nil {
 			return fmt.Errorf("state[%s] must be an int", key)
 		} else {
 			c.state.Store(key, vi+amount)
@@ -141,7 +142,7 @@ func (c *DefaultContext) IncrCounter(key string, amount int) error {
 
 func (c *DefaultContext) GetCounter(key string) (int, error) {
 	if v, ok := c.state.Load(key); ok {
-		if vi, err := common.ToInt(v, common.STRICT); err != nil {
+		if vi, err := cast.ToInt(v, cast.STRICT); err != nil {
 			return 0, fmt.Errorf("state[%s] is not a number, but %v", key, v)
 		} else {
 			return vi, nil
@@ -171,7 +172,7 @@ func (c *DefaultContext) DeleteState(key string) error {
 }
 
 func (c *DefaultContext) Snapshot() error {
-	c.snapshot = common.SyncMapToMap(c.state)
+	c.snapshot = cast.SyncMapToMap(c.state)
 	return nil
 }
 

xstream/contexts/default_test.go

@@ -1,9 +1,9 @@
 package contexts
 
 import (
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
-	"github.com/emqx/kuiper/xstream/states"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/topo/states"
+	"github.com/emqx/kuiper/pkg/api"
 	"log"
 	"os"
 	"path"
@@ -77,13 +77,13 @@ func TestState(t *testing.T) {
 }
 
 func cleanStateData() {
-	dbDir, err := common.GetDataLoc()
+	dbDir, err := conf.GetDataLoc()
 	if err != nil {
 		log.Panic(err)
 	}
 	c := path.Join(dbDir, "checkpoints")
 	err = os.RemoveAll(c)
 	if err != nil {
-		common.Log.Error(err)
+		conf.Log.Error(err)
 	}
 }

xstream/contexts/func_context.go

@@ -2,7 +2,7 @@ package contexts
 
 import (
 	"fmt"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/pkg/api"
 )
 
 type DefaultFuncContext struct {

xstream/extensions/edgex_source.go

@@ -9,8 +9,10 @@ import (
 	"github.com/edgexfoundry/go-mod-core-contracts/v2/dtos"
 	"github.com/edgexfoundry/go-mod-messaging/v2/messaging"
 	"github.com/edgexfoundry/go-mod-messaging/v2/pkg/types"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
+	"github.com/emqx/kuiper/pkg/message"
 	"strconv"
 	"strings"
 )
@@ -23,7 +25,7 @@ type EdgexSource struct {
 
 func (es *EdgexSource) Configure(_ string, props map[string]interface{}) error {
 	if f, ok := props["format"]; ok {
-		if f != common.FORMAT_JSON {
+		if f != message.FormatJson {
 			return fmt.Errorf("edgex source only supports `json` format")
 		}
 	}
@@ -58,10 +60,10 @@ func (es *EdgexSource) Configure(_ string, props map[string]interface{}) error {
 	if ops, ok := props["optional"]; ok {
 		if ops1, ok1 := ops.(map[string]interface{}); ok1 {
 			for k, v := range ops1 {
-				if cv, ok := CastToString(v); ok {
+				if cv, err := cast.ToString(v, cast.CONVERT_ALL); err == nil {
 					optional[k] = cv
 				} else {
-					common.Log.Infof("Cannot convert configuration %s: %s to string type.\n", k, v)
+					conf.Log.Infof("Cannot convert configuration %s: %s to string type.\n", k, v)
 				}
 			}
 		}
@@ -88,7 +90,7 @@ func printConf(mbconf types.MessageBusConfig) {
 		}
 	}
 	mbconf.Optional = printableOptional
-	common.Log.Infof("Use configuration for edgex messagebus %v", mbconf)
+	conf.Log.Infof("Use configuration for edgex messagebus %v", mbconf)
 }
 
 func (es *EdgexSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple, errCh chan<- error) {

xstream/extensions/edgex_source_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/edgexfoundry/go-mod-core-contracts/v2/dtos"
 	"github.com/edgexfoundry/go-mod-core-contracts/v2/models"
 	"github.com/edgexfoundry/go-mod-messaging/v2/pkg/types"
-	"github.com/emqx/kuiper/common"
+	"github.com/emqx/kuiper/internal/conf"
 	"math"
 	"reflect"
 	"testing"
@@ -60,7 +60,7 @@ func TestGetValue_IntFloat(t *testing.T) {
 
 	dtoe := dtos.FromEventModelToDTO(testEvent)
 	for _, r := range dtoe.Readings {
-		if v, e := es.getValue(r, common.Log); e != nil {
+		if v, e := es.getValue(r, conf.Log); e != nil {
 			t.Errorf("%s", e)
 		} else {
 			expectOne(t, v)
@@ -68,7 +68,7 @@ func TestGetValue_IntFloat(t *testing.T) {
 	}
 
 	r1 := dtos.BaseReading{ResourceName: "i8", ValueType: typeMap["i8"], SimpleReading: dtos.SimpleReading{Value: "10796529505058023104"}}
-	if v, e := es.getValue(r1, common.Log); e != nil {
+	if v, e := es.getValue(r1, conf.Log); e != nil {
 		t.Errorf("%s", e)
 	} else {
 		if v1, ok := v.(uint64); ok {
@@ -79,7 +79,7 @@ func TestGetValue_IntFloat(t *testing.T) {
 	}
 
 	r2 := dtos.BaseReading{ResourceName: "f1", ValueType: typeMap["f1"], SimpleReading: dtos.SimpleReading{Value: "3.14"}}
-	if v, e := es.getValue(r2, common.Log); e != nil {
+	if v, e := es.getValue(r2, conf.Log); e != nil {
 		t.Errorf("%s", e)
 	} else {
 		if v1, ok := v.(float64); ok {
@@ -112,7 +112,7 @@ func TestGetValue_IntFloatArr(t *testing.T) {
 
 	dtoe := dtos.FromEventModelToDTO(testEvent)
 	for i, r := range dtoe.Readings {
-		if v, e := es.getValue(r, common.Log); e != nil {
+		if v, e := es.getValue(r, conf.Log); e != nil {
 			t.Errorf("%s", e)
 		} else {
 			checkArray(t, i, v)
@@ -120,7 +120,7 @@ func TestGetValue_IntFloatArr(t *testing.T) {
 	}
 
 	r1 := dtos.BaseReading{ResourceName: "ia8", ValueType: typeMap["ia8"], SimpleReading: dtos.SimpleReading{Value: `[10796529505058023104, 10796529505058023105]`}}
-	if v, e := es.getValue(r1, common.Log); e != nil {
+	if v, e := es.getValue(r1, conf.Log); e != nil {
 		t.Errorf("%s", e)
 	} else {
 		if v1, ok := v.([]uint64); ok {
@@ -133,7 +133,7 @@ func TestGetValue_IntFloatArr(t *testing.T) {
 	}
 
 	rf_00 := dtos.BaseReading{ResourceName: "fa1", ValueType: typeMap["fa1"], SimpleReading: dtos.SimpleReading{Value: `[3.14, 2.71828]`}}
-	if v, e := es.getValue(rf_00, common.Log); e != nil {
+	if v, e := es.getValue(rf_00, conf.Log); e != nil {
 		t.Errorf("%s", e)
 	} else {
 		if v1, ok := v.([]float64); ok {
@@ -183,7 +183,7 @@ func TestGetValue_Float(t *testing.T) {
 
 	dtoe := dtos.FromEventModelToDTO(testEvent)
 	for _, r := range dtoe.Readings {
-		if v, e := es.getValue(r, common.Log); e != nil {
+		if v, e := es.getValue(r, conf.Log); e != nil {
 			t.Errorf("%s", e)
 		} else {
 			expectPi(t, v)
@@ -206,7 +206,7 @@ func TestGetValue_Bool(t *testing.T) {
 	trues := []string{"1", "t", "T", "true", "TRUE", "True"}
 	for _, v := range trues {
 		r1 := dtos.BaseReading{ResourceName: "b1", ValueType: typeMap["b1"], SimpleReading: dtos.SimpleReading{Value: v}}
-		if v, e := es.getValue(r1, common.Log); e != nil {
+		if v, e := es.getValue(r1, conf.Log); e != nil {
 			t.Errorf("%s", e)
 		} else {
 			expectTrue(t, v)
@@ -214,7 +214,7 @@ func TestGetValue_Bool(t *testing.T) {
 	}
 
 	r1 := dtos.BaseReading{ResourceName: "b1", ValueType: typeMap["b1"], SimpleReading: dtos.SimpleReading{Value: "TRue"}}
-	if _, e := es.getValue(r1, common.Log); e == nil {
+	if _, e := es.getValue(r1, conf.Log); e == nil {
 		t.Errorf("%s", e)
 	}
 
@@ -222,7 +222,7 @@ func TestGetValue_Bool(t *testing.T) {
 	falses := []string{"0", "f", "F", "false", "FALSE", "False"}
 	for _, v := range falses {
 		r1 := dtos.BaseReading{ResourceName: "b1", ValueType: typeMap["b1"], SimpleReading: dtos.SimpleReading{Value: v}}
-		if v, e := es.getValue(r1, common.Log); e != nil {
+		if v, e := es.getValue(r1, conf.Log); e != nil {
 			t.Errorf("%s", e)
 		} else {
 			expectFalse(t, v)
@@ -230,7 +230,7 @@ func TestGetValue_Bool(t *testing.T) {
 	}
 
 	r1 = dtos.BaseReading{ResourceName: "b1", ValueType: typeMap["b1"], SimpleReading: dtos.SimpleReading{Value: "FAlse"}}
-	if _, e := es.getValue(r1, common.Log); e == nil {
+	if _, e := es.getValue(r1, conf.Log); e == nil {
 		t.Errorf("%s", e)
 	}
 }
@@ -257,7 +257,7 @@ func expectFalse(t *testing.T, expected interface{}) {
 
 func TestWrongType(t *testing.T) {
 	r1 := dtos.BaseReading{ResourceName: "f", ValueType: "FLOAT", SimpleReading: dtos.SimpleReading{Value: "100"}}
-	if v, _ := es.getValue(r1, common.Log); v != "100" {
+	if v, _ := es.getValue(r1, conf.Log); v != "100" {
 		t.Errorf("Expected 100, but it's %s!", v)
 	}
 }
@@ -292,27 +292,12 @@ func TestWrongValue(t *testing.T) {
 
 	dtoe := dtos.FromEventModelToDTO(testEvent)
 	for _, v := range dtoe.Readings {
-		if _, e := es.getValue(v, common.Log); e == nil {
+		if _, e := es.getValue(v, conf.Log); e == nil {
 			t.Errorf("Expected an error!")
 		}
 	}
 }
 
-func TestCastToString(t *testing.T) {
-	if v, ok := CastToString(12); v != "12" || !ok {
-		t.Errorf("Failed to cast int.")
-	}
-	if v, ok := CastToString(true); v != "true" || !ok {
-		t.Errorf("Failed to cast bool.")
-	}
-	if v, ok := CastToString("hello"); v != "hello" || !ok {
-		t.Errorf("Failed to cast string.")
-	}
-	if v, ok := CastToString(12.3); v != "12.30" || !ok {
-		t.Errorf("Failed to cast float.")
-	}
-}
-
 func TestPrintConf(t *testing.T) {
 	expMbconf := types.MessageBusConfig{SubscribeHost: types.HostInfo{Protocol: "tcp", Host: "127.0.0.1", Port: 6625}, Type: "mbus", Optional: map[string]string{
 		"proa":     "proa",
@@ -333,7 +318,7 @@ func TestPrintConf(t *testing.T) {
 func TestGetValue_Binary(t *testing.T) {
 	ev := []byte("Hello World")
 	r1 := dtos.BaseReading{ResourceName: "bin", ValueType: v2.ValueTypeBinary, BinaryReading: dtos.BinaryReading{MediaType: "application/text", BinaryValue: ev}}
-	if v, e := es.getValue(r1, common.Log); e != nil {
+	if v, e := es.getValue(r1, conf.Log); e != nil {
 		t.Errorf("%s", e)
 	} else if !reflect.DeepEqual(ev, v) {
 		t.Errorf("result mismatch, expect %v, but got %v", ev, v)

xstream/extensions/file_source.go

@@ -3,8 +3,10 @@ package extensions
 import (
 	"errors"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/filex"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
 	"os"
 	"path"
 	"path/filepath"
@@ -42,7 +44,7 @@ func (fs *FileSource) Close(ctx api.StreamContext) error {
 
 func (fs *FileSource) Configure(fileName string, props map[string]interface{}) error {
 	cfg := &FileSourceConfig{}
-	err := common.MapToStruct(props, cfg)
+	err := cast.MapToStruct(props, cfg)
 	if err != nil {
 		return fmt.Errorf("read properties %v fail with error: %v", props, err)
 	}
@@ -59,7 +61,7 @@ func (fs *FileSource) Configure(fileName string, props map[string]interface{}) e
 		return errors.New("file name must be specified")
 	}
 	if !filepath.IsAbs(cfg.Path) {
-		cfg.Path, err = common.GetLoc(cfg.Path)
+		cfg.Path, err = conf.GetLoc(cfg.Path)
 		if err != nil {
 			return fmt.Errorf("invalid path %s", cfg.Path)
 		}
@@ -90,7 +92,7 @@ func (fs *FileSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTupl
 		for {
 			select {
 			case <-ticker.C:
-				logger.Debugf("Load file source again at %v", common.GetNowInMilli())
+				logger.Debugf("Load file source again at %v", conf.GetNowInMilli())
 				err := fs.Load(ctx, consumer)
 				if err != nil {
 					errCh <- err
@@ -108,7 +110,7 @@ func (fs *FileSource) Load(ctx api.StreamContext, consumer chan<- api.SourceTupl
 	case JSON_TYPE:
 		ctx.GetLogger().Debugf("Start to load from file %s", fs.file)
 		resultMap := make([]map[string]interface{}, 0)
-		err := common.ReadJsonUnmarshal(fs.file, &resultMap)
+		err := filex.ReadJsonUnmarshal(fs.file, &resultMap)
 		if err != nil {
 			return fmt.Errorf("loaded %s, check error %s", fs.file, err)
 		}

xstream/extensions/httppull_source.go

@@ -4,8 +4,11 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"github.com/emqx/kuiper/common"
-	"github.com/emqx/kuiper/xstream/api"
+	"github.com/emqx/kuiper/internal/conf"
+	"github.com/emqx/kuiper/internal/pkg/httpx"
+	"github.com/emqx/kuiper/pkg/api"
+	"github.com/emqx/kuiper/pkg/cast"
+	"github.com/emqx/kuiper/pkg/message"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -92,7 +95,7 @@ func (hps *HTTPPullSource) Configure(device string, props map[string]interface{}
 		}
 	}
 
-	hps.messageFormat = common.FORMAT_JSON
+	hps.messageFormat = message.FormatJson
 	if c, ok := props["format"]; ok {
 		if c1, ok1 := c.(string); ok1 {
 			hps.messageFormat = c1
@@ -113,7 +116,7 @@ func (hps *HTTPPullSource) Configure(device string, props map[string]interface{}
 	if h, ok := props["headers"]; ok {
 		if h1, ok1 := h.(map[string]interface{}); ok1 {
 			for k, v := range h1 {
-				if v1, ok2 := CastToString(v); ok2 {
+				if v1, err := cast.ToString(v, cast.CONVERT_ALL); err == nil {
 					hps.headers[k] = v1
 				}
 			}
@@ -122,7 +125,7 @@ func (hps *HTTPPullSource) Configure(device string, props map[string]interface{}
 		}
 	}
 
-	common.Log.Debugf("Initialized with configurations %#v.", hps)
+	conf.Log.Debugf("Initialized with configurations %#v.", hps)
 	return nil
 }
 
@@ -151,7 +154,7 @@ func (hps *HTTPPullSource) initTimerPull(ctx api.StreamContext, consumer chan<-
 	for {
 		select {
 		case <-ticker.C:
-			if resp, e := common.Send(logger, hps.client, hps.bodyType, hps.method, hps.url, hps.headers, true, []byte(hps.body)); e != nil {
+			if resp, e := httpx.Send(logger, hps.client, hps.bodyType, hps.method, hps.url, hps.headers, true, []byte(hps.body)); e != nil {
 				logger.Warnf("Found error %s when trying to reach %v ", e, hps)
 			} else {
 				logger.Debugf("rest sink got response %v", resp)
@@ -174,7 +177,7 @@ func (hps *HTTPPullSource) initTimerPull(ctx api.StreamContext, consumer chan<-
 					}
 				}
 
-				result, e := common.MessageDecode(c, hps.messageFormat)
+				result, e := message.Decode(c, hps.messageFormat)
 				meta := make(map[string]interface{})
 				if e != nil {
 					logger.Errorf("Invalid data format, cannot decode %s to %s format with error %s", string(c), hps.messageFormat, e)