refactor(*): refactor the binding of io(source/sink) and function

1. Manage the function binding from built-in, native plugin and external service in one place
2. Manage the io binding from built-in, native plugin in one place
3. Separate the plugin meta reading from the native plugin manager into binder. As all io/function provider has metas to read.

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

extensions.mod

@@ -38,7 +38,7 @@ require (
 	github.com/tebeka/strftime v0.1.5 // indirect
 	github.com/ugorji/go/codec v1.2.5
 	github.com/urfave/cli v1.22.0
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0

extensions.sum

@@ -268,9 +268,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -291,16 +290,14 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/tebeka/strftime v0.1.5 // indirect
 	github.com/ugorji/go/codec v1.2.5
 	github.com/urfave/cli v1.22.0
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
+	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0

go.sum

@@ -273,9 +273,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
-golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -296,16 +295,14 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

internal/topo/node/sources_with_edgex.go

@@ -12,32 +12,26 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build edgex
-// +build !test
+package binder
 
-package node
+import "github.com/lf-edge/ekuiper/pkg/api"
 
-import (
-	"github.com/lf-edge/ekuiper/internal/topo/sink"
-	"github.com/lf-edge/ekuiper/internal/topo/source"
-	"github.com/lf-edge/ekuiper/pkg/api"
-)
+type SourceFactory interface {
+	Source(name string) (api.Source, error)
+}
+
+type SinkFactory interface {
+	Sink(name string) (api.Sink, error)
+}
 
-func getSource(t string) (api.Source, error) {
-	if t == "edgex" {
-		return &source.EdgexSource{}, nil
-	}
-	return doGetSource(t)
+type FuncFactory interface {
+	Function(name string) (api.Function, error)
+	// HasFunctionSet Some functions are bundled together into a plugin which shares the same json file.
+	// This function can return if the function set name exists.
+	HasFunctionSet(funcName string) bool
 }
 
-func getSink(name string, action map[string]interface{}) (api.Sink, error) {
-	if name == "edgex" {
-		s := &sink.EdgexMsgBusSink{}
-		if err := s.Configure(action); err != nil {
-			return nil, err
-		} else {
-			return s, nil
-		}
-	}
-	return doGetSink(name, action)
+type FactoryEntry struct {
+	Name    string
+	Factory interface{}
 }

internal/binder/function/binder.go

@@ -0,0 +1,89 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package function
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"github.com/lf-edge/ekuiper/pkg/errorx"
+)
+
+var ( // init once and read only
+	funcFactories      []binder.FuncFactory
+	funcFactoriesNames []string
+)
+
+func init() {
+	f := binder.FactoryEntry{
+		Name:    "built-in",
+		Factory: GetManager(),
+	}
+	applyFactory(f)
+}
+
+// Initialize Only call once when server starts
+func Initialize(factories []binder.FactoryEntry) error {
+	for _, f := range factories {
+		applyFactory(f)
+	}
+	return nil
+}
+
+func applyFactory(f binder.FactoryEntry) {
+	if s, ok := f.Factory.(binder.FuncFactory); ok {
+		funcFactories = append(funcFactories, s)
+		funcFactoriesNames = append(funcFactoriesNames, f.Name)
+	}
+}
+
+func Function(name string) (api.Function, error) {
+	e := make(errorx.MultiError)
+	for i, sf := range funcFactories {
+		r, err := sf.Function(name)
+		if err != nil {
+			e[funcFactoriesNames[i]] = err
+		}
+		if r != nil {
+			return r, e.GetError()
+		}
+	}
+	return nil, e.GetError()
+}
+
+func HasFunctionSet(name string) bool {
+	for _, sf := range funcFactories {
+		r := sf.HasFunctionSet(name)
+		if r {
+			return r
+		}
+	}
+	return false
+}
+
+type multiAggFunc interface {
+	IsAggregateWithName(name string) bool
+}
+
+func IsAggFunc(funcName string) bool {
+	f, _ := Function(funcName)
+	if f != nil {
+		if mf, ok := f.(multiAggFunc); ok {
+			return mf.IsAggregateWithName(funcName)
+		} else {
+			return f.IsAggregate()
+		}
+	}
+	return false
+}

internal/binder/function/funcs_agg.go

@@ -0,0 +1,312 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package function
+
+import (
+	"fmt"
+	"strings"
+)
+
+func aggCall(name string, args []interface{}) (interface{}, bool) {
+	lowerName := strings.ToLower(name)
+	switch lowerName {
+	case "avg":
+		arg0 := args[0].([]interface{})
+		c := getCount(arg0)
+		if c > 0 {
+			v := getFirstValidArg(arg0)
+			switch v.(type) {
+			case int, int64:
+				if r, err := sliceIntTotal(arg0); err != nil {
+					return err, false
+				} else {
+					return r / c, true
+				}
+			case float64:
+				if r, err := sliceFloatTotal(arg0); err != nil {
+					return err, false
+				} else {
+					return r / float64(c), true
+				}
+			case nil:
+				return nil, true
+			default:
+				return fmt.Errorf("run avg function error: found invalid arg %[1]T(%[1]v)", v), false
+			}
+		}
+		return 0, true
+	case "count":
+		arg0 := args[0].([]interface{})
+		return getCount(arg0), true
+	case "max":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0 {
+			v := getFirstValidArg(arg0)
+			switch t := v.(type) {
+			case int:
+				if r, err := sliceIntMax(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case int64:
+				if r, err := sliceIntMax(arg0, int(t)); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case float64:
+				if r, err := sliceFloatMax(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case string:
+				if r, err := sliceStringMax(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case nil:
+				return nil, true
+			default:
+				return fmt.Errorf("run max function error: found invalid arg %[1]T(%[1]v)", v), false
+			}
+		}
+		return fmt.Errorf("run max function error: empty data"), false
+	case "min":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0 {
+			v := getFirstValidArg(arg0)
+			switch t := v.(type) {
+			case int:
+				if r, err := sliceIntMin(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case int64:
+				if r, err := sliceIntMin(arg0, int(t)); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case float64:
+				if r, err := sliceFloatMin(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case string:
+				if r, err := sliceStringMin(arg0, t); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case nil:
+				return nil, true
+			default:
+				return fmt.Errorf("run min function error: found invalid arg %[1]T(%[1]v)", v), false
+			}
+		}
+		return fmt.Errorf("run min function error: empty data"), false
+	case "sum":
+		arg0 := args[0].([]interface{})
+		if len(arg0) > 0 {
+			v := getFirstValidArg(arg0)
+			switch v.(type) {
+			case int, int64:
+				if r, err := sliceIntTotal(arg0); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case float64:
+				if r, err := sliceFloatTotal(arg0); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			case nil:
+				return nil, true
+			default:
+				return fmt.Errorf("run sum function error: found invalid arg %[1]T(%[1]v)", v), false
+			}
+		}
+		return 0, true
+	case "collect":
+		return args[0], true
+	case "deduplicate":
+		v1, ok1 := args[0].([]interface{})
+		v2, ok2 := args[1].([]interface{})
+		v3a, ok3 := args[2].([]interface{})
+
+		if ok1 && ok2 && ok3 && len(v3a) > 0 {
+			v3, ok4 := getFirstValidArg(v3a).(bool)
+			if ok4 {
+				if r, err := dedup(v1, v2, v3); err != nil {
+					return err, false
+				} else {
+					return r, true
+				}
+			}
+		}
+		return fmt.Errorf("Invalid argument type found."), false
+	default:
+		return fmt.Errorf("Unknown aggregate function name."), false
+	}
+}
+
+func getCount(s []interface{}) int {
+	c := 0
+	for _, v := range s {
+		if v != nil {
+			c++
+		}
+	}
+	return c
+}
+
+func getFirstValidArg(s []interface{}) interface{} {
+	for _, v := range s {
+		if v != nil {
+			return v
+		}
+	}
+	return nil
+}
+
+func sliceIntTotal(s []interface{}) (int, error) {
+	var total int
+	for _, v := range s {
+		if vi, ok := v.(int); ok {
+			total += vi
+		} else if v != nil {
+			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
+		}
+	}
+	return total, nil
+}
+
+func sliceFloatTotal(s []interface{}) (float64, error) {
+	var total float64
+	for _, v := range s {
+		if vf, ok := v.(float64); ok {
+			total += vf
+		} else if v != nil {
+			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
+		}
+	}
+	return total, nil
+}
+func sliceIntMax(s []interface{}, max int) (int, error) {
+	for _, v := range s {
+		if vi, ok := v.(int); ok {
+			if max < vi {
+				max = vi
+			}
+		} else if v != nil {
+			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
+		}
+	}
+	return max, nil
+}
+func sliceFloatMax(s []interface{}, max float64) (float64, error) {
+	for _, v := range s {
+		if vf, ok := v.(float64); ok {
+			if max < vf {
+				max = vf
+			}
+		} else if v != nil {
+			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
+		}
+	}
+	return max, nil
+}
+
+func sliceStringMax(s []interface{}, max string) (string, error) {
+	for _, v := range s {
+		if vs, ok := v.(string); ok {
+			if max < vs {
+				max = vs
+			}
+		} else if v != nil {
+			return "", fmt.Errorf("requires string but found %[1]T(%[1]v)", v)
+		}
+	}
+	return max, nil
+}
+func sliceIntMin(s []interface{}, min int) (int, error) {
+	for _, v := range s {
+		if vi, ok := v.(int); ok {
+			if min > vi {
+				min = vi
+			}
+		} else if v != nil {
+			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
+		}
+	}
+	return min, nil
+}
+func sliceFloatMin(s []interface{}, min float64) (float64, error) {
+	for _, v := range s {
+		if vf, ok := v.(float64); ok {
+			if min > vf {
+				min = vf
+			}
+		} else if v != nil {
+			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
+		}
+	}
+	return min, nil
+}
+
+func sliceStringMin(s []interface{}, min string) (string, error) {
+	for _, v := range s {
+		if vs, ok := v.(string); ok {
+			if min < vs {
+				min = vs
+			}
+		} else if v != nil {
+			return "", fmt.Errorf("requires string but found %[1]T(%[1]v)", v)
+		}
+	}
+	return min, nil
+}
+
+func dedup(r []interface{}, col []interface{}, all bool) (interface{}, error) {
+	keyset := make(map[string]bool)
+	result := make([]interface{}, 0)
+	for i, m := range col {
+		key := fmt.Sprintf("%v", m)
+		if _, ok := keyset[key]; !ok {
+			if all {
+				result = append(result, r[i])
+			} else if i == len(col)-1 {
+				result = append(result, r[i])
+			}
+			keyset[key] = true
+		}
+	}
+	if !all {
+		if len(result) == 0 {
+			return nil, nil
+		} else {
+			return result[0], nil
+		}
+	} else {
+		return result, nil
+	}
+}

internal/xsql/funcsAstValidator.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package xsql
+package function
 
 import (
 	"fmt"
@@ -26,32 +26,23 @@ type AllowTypes struct {
 
 func validateFuncs(funcName string, args []ast.Expr) error {
 	lowerName := strings.ToLower(funcName)
-	switch ast.FuncFinderSingleton().FuncType(lowerName) {
-	case ast.NotFoundFunc:
-		nf, _, err := parserFuncRuntime.Get(funcName)
-		if err != nil {
-			return fmt.Errorf("error getting function %s: %v", funcName, err)
-		}
-		var targs []interface{}
-		for _, arg := range args {
-			targs = append(targs, arg)
-		}
-		return nf.Validate(targs)
-	case ast.AggFunc:
+	switch getFuncType(lowerName) {
+	case AggFunc:
 		return validateAggFunc(lowerName, args)
-	case ast.MathFunc:
+	case MathFunc:
 		return validateMathFunc(lowerName, args)
-	case ast.ConvFunc:
+	case ConvFunc:
 		return validateConvFunc(lowerName, args)
-	case ast.StrFunc:
+	case StrFunc:
 		return validateStrFunc(lowerName, args)
-	case ast.HashFunc:
+	case HashFunc:
 		return validateHashFunc(lowerName, args)
-	case ast.JsonFunc:
+	case JsonFunc:
 		return validateJsonFunc(lowerName, args)
-	case ast.OtherFunc:
+	case OtherFunc:
 		return validateOtherFunc(lowerName, args)
 	default:
+		// should not happen
 		return fmt.Errorf("unkndow function %s", lowerName)
 	}
 }

internal/xsql/funcsMath.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package xsql
+package function
 
 import (
 	"fmt"

internal/xsql/funcsMisc.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package xsql
+package function
 
 import (
 	"context"

internal/xsql/funcsStr.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package xsql
+package function
 
 import (
 	"bytes"

pkg/ast/functions.go

@@ -12,18 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ast
+package function
 
 import (
+	"fmt"
 	"github.com/lf-edge/ekuiper/pkg/api"
+	"github.com/lf-edge/ekuiper/pkg/ast"
 	"strings"
-	"sync"
 )
 
-type FuncType int
+type funcType int
 
 const (
-	NotFoundFunc FuncType = iota - 1
+	NotFoundFunc funcType = iota - 1
 	AggFunc
 	MathFunc
 	StrFunc
@@ -93,64 +94,86 @@ var otherFuncMap = map[string]string{"isnull": "",
 	"window_end":   "",
 }
 
-type FuncRuntime interface {
-	Get(name string) (api.Function, api.FunctionContext, error)
+func getFuncType(name string) funcType {
+	for i, m := range maps {
+		if _, ok := m[strings.ToLower(name)]; ok {
+			return funcType(i)
+		}
+	}
+	return NotFoundFunc
 }
 
-var (
-	once sync.Once
-	ff   *FuncFinder
-)
+type funcExecutor struct{}
 
-// FuncFinder Singleton, must be initiated when starting
-type FuncFinder struct {
-	runtime FuncRuntime
-}
-
-// InitFuncFinder must be called when starting
-func InitFuncFinder(runtime FuncRuntime) {
-	once.Do(func() {
-		ff = &FuncFinder{runtime: runtime}
-	})
-	ff.runtime = runtime
-}
-
-// FuncFinderSingleton must be inited before calling this
-func FuncFinderSingleton() *FuncFinder {
-	return ff
-}
-
-func (ff *FuncFinder) IsAggFunc(f *Call) bool {
-	fn := strings.ToLower(f.Name)
-	if _, ok := aggFuncMap[fn]; ok {
-		return true
-	} else if _, ok := strFuncMap[fn]; ok {
-		return false
-	} else if _, ok := convFuncMap[fn]; ok {
-		return false
-	} else if _, ok := hashFuncMap[fn]; ok {
-		return false
-	} else if _, ok := otherFuncMap[fn]; ok {
-		return false
-	} else if _, ok := mathFuncMap[fn]; ok {
-		return false
-	} else {
-		if nf, _, err := ff.runtime.Get(f.Name); err == nil {
-			if nf.IsAggregate() {
-				//Add cache
-				aggFuncMap[fn] = ""
-				return true
-			}
+func (f *funcExecutor) ValidateWithName(args []ast.Expr, name string) error {
+	var eargs []ast.Expr
+	for _, arg := range args {
+		if t, ok := arg.(ast.Expr); ok {
+			eargs = append(eargs, t)
+		} else {
+			// should never happen
+			return fmt.Errorf("receive invalid arg %v", arg)
 		}
 	}
+	return validateFuncs(name, eargs)
+}
+
+func (f *funcExecutor) Validate(_ []interface{}) error {
+	return fmt.Errorf("unknow name")
+}
+
+func (f *funcExecutor) Exec(_ []interface{}, _ api.FunctionContext) (interface{}, bool) {
+	return fmt.Errorf("unknow name"), false
+}
+
+func (f *funcExecutor) ExecWithName(args []interface{}, _ api.FunctionContext, name string) (interface{}, bool) {
+	lowerName := strings.ToLower(name)
+	switch getFuncType(lowerName) {
+	case AggFunc:
+		return aggCall(lowerName, args)
+	case MathFunc:
+		return mathCall(lowerName, args)
+	case ConvFunc:
+		return convCall(lowerName, args)
+	case StrFunc:
+		return strCall(lowerName, args)
+	case HashFunc:
+		return hashCall(lowerName, args)
+	case JsonFunc:
+		return jsonCall(lowerName, args)
+	case OtherFunc:
+		return otherCall(lowerName, args)
+	}
+	return fmt.Errorf("unknow name"), false
+}
+
+func (f *funcExecutor) IsAggregate() bool {
 	return false
 }
 
-func (ff *FuncFinder) FuncType(name string) FuncType {
-	for i, m := range maps {
-		if _, ok := m[strings.ToLower(name)]; ok {
-			return FuncType(i)
-		}
+func (f *funcExecutor) IsAggregateWithName(name string) bool {
+	lowerName := strings.ToLower(name)
+	return getFuncType(lowerName) == AggFunc
+}
+
+var staticFuncExecutor = &funcExecutor{}
+
+type Manager struct{}
+
+func (m *Manager) Function(name string) (api.Function, error) {
+	ft := getFuncType(name)
+	if ft != NotFoundFunc {
+		return staticFuncExecutor, nil
 	}
-	return NotFoundFunc
+	return nil, nil
+}
+
+func (m *Manager) HasFunctionSet(name string) bool {
+	return name == "internal"
+}
+
+var m = &Manager{}
+
+func GetManager() *Manager {
+	return m
 }

internal/binder/io/binder.go

@@ -0,0 +1,82 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package io
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"github.com/lf-edge/ekuiper/pkg/errorx"
+)
+
+var ( // init once and read only
+	sourceFactories      []binder.SourceFactory
+	sourceFactoriesNames []string
+	sinkFactories        []binder.SinkFactory
+	sinkFactoriesNames   []string
+)
+
+func init() {
+	f := binder.FactoryEntry{
+		Name:    "built-in",
+		Factory: GetManager(),
+	}
+	applyFactory(f)
+}
+
+func Initialize(factories []binder.FactoryEntry) error {
+	for _, f := range factories {
+		applyFactory(f)
+	}
+	return nil
+}
+
+func applyFactory(f binder.FactoryEntry) {
+	if s, ok := f.Factory.(binder.SourceFactory); ok {
+		sourceFactories = append(sourceFactories, s)
+		sourceFactoriesNames = append(sourceFactoriesNames, f.Name)
+	}
+	if s, ok := f.Factory.(binder.SinkFactory); ok {
+		sinkFactories = append(sinkFactories, s)
+		sinkFactoriesNames = append(sinkFactoriesNames, f.Name)
+	}
+}
+
+func Source(name string) (api.Source, error) {
+	e := make(errorx.MultiError)
+	for i, sf := range sourceFactories {
+		r, err := sf.Source(name)
+		if err != nil {
+			e[sourceFactoriesNames[i]] = err
+		}
+		if r != nil {
+			return r, e.GetError()
+		}
+	}
+	return nil, e.GetError()
+}
+
+func Sink(name string) (api.Sink, error) {
+	e := make(errorx.MultiError)
+	for i, sf := range sinkFactories {
+		r, err := sf.Sink(name)
+		if err != nil {
+			e[sinkFactoriesNames[i]] = err
+		}
+		if r != nil {
+			return r, e.GetError()
+		}
+	}
+	return nil, e.GetError()
+}

internal/binder/io/builtin.go

@@ -0,0 +1,61 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package io
+
+import (
+	"github.com/lf-edge/ekuiper/internal/topo/sink"
+	"github.com/lf-edge/ekuiper/internal/topo/source"
+	"github.com/lf-edge/ekuiper/pkg/api"
+)
+
+type NewSourceFunc func() api.Source
+type NewSinkFunc func() api.Sink
+
+var (
+	sources = map[string]NewSourceFunc{
+		"mqtt":     func() api.Source { return &source.MQTTSource{} },
+		"httppull": func() api.Source { return &source.HTTPPullSource{} },
+		"file":     func() api.Source { return &source.FileSource{} },
+	}
+	sinks = map[string]NewSinkFunc{
+		"log":         sink.NewLogSink,
+		"logToMemory": sink.NewLogSinkToMemory,
+		"mqtt":        func() api.Sink { return &sink.MQTTSink{} },
+		"rest":        func() api.Sink { return &sink.RestSink{} },
+		"nop":         func() api.Sink { return &sink.NopSink{} },
+	}
+)
+
+type Manager struct{}
+
+func (m *Manager) Source(name string) (api.Source, error) {
+	if s, ok := sources[name]; ok {
+		return s(), nil
+	}
+	return nil, nil
+}
+
+func (m *Manager) Sink(name string) (api.Sink, error) {
+	if s, ok := sinks[name]; ok {
+		return s(), nil
+	}
+	return nil, nil
+}
+
+var m = &Manager{}
+
+func GetManager() *Manager {
+	return m
+}

internal/topo/node/sources_for_test_without_edgex.go

@@ -12,23 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !edgex
-// +build test
+//go:build edgex
+// +build edgex
 
-package node
+package io
 
 import (
-	"github.com/lf-edge/ekuiper/internal/topo/topotest/mocknode"
+	"github.com/lf-edge/ekuiper/internal/topo/sink"
+	"github.com/lf-edge/ekuiper/internal/topo/source"
 	"github.com/lf-edge/ekuiper/pkg/api"
 )
 
-func getSource(t string) (api.Source, error) {
-	if t == "mock" {
-		return &mocknode.MockSource{}, nil
-	}
-	return doGetSource(t)
-}
-
-func getSink(name string, action map[string]interface{}) (api.Sink, error) {
-	return doGetSink(name, action)
+func init() {
+	sources["edgex"] = func() api.Source { return &source.EdgexSource{} }
+	sinks["edgex"] = func() api.Sink { return &sink.EdgexMsgBusSink{} }
 }

internal/topo/node/sources_without_edgex.go

@@ -12,19 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// +build !edgex
-// +build !test
+//go:build test
+// +build test
 
-package node
+package io
 
 import (
+	"github.com/lf-edge/ekuiper/internal/topo/topotest/mocknode"
 	"github.com/lf-edge/ekuiper/pkg/api"
 )
 
-func getSource(t string) (api.Source, error) {
-	return doGetSource(t)
-}
-
-func getSink(name string, action map[string]interface{}) (api.Sink, error) {
-	return doGetSink(name, action)
+func init() {
+	sources["mock"] = func() api.Source { return &mocknode.MockSource{} }
 }

internal/binder/meta/bind.go

@@ -0,0 +1,45 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package meta
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/meta"
+)
+
+func Bind() {
+	if err := meta.ReadSourceMetaDir(func(name string) bool {
+		s, _ := io.Source(name)
+		return s != nil
+	}); nil != err {
+		conf.Log.Errorf("readSourceMetaDir:%v", err)
+	}
+	if err := meta.ReadSinkMetaDir(func(name string) bool {
+		s, _ := io.Sink(name)
+		return s != nil
+	}); nil != err {
+		conf.Log.Errorf("readSinkMetaDir:%v", err)
+	}
+	if err := meta.ReadFuncMetaDir(func(name string) bool {
+		return function.HasFunctionSet(name)
+	}); nil != err {
+		conf.Log.Errorf("readFuncMetaDir:%v", err)
+	}
+	if err := meta.ReadUiMsgDir(); nil != err {
+		conf.Log.Errorf("readUiMsgDir:%v", err)
+	}
+}

internal/plugin/funcMeta.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	"fmt"
@@ -46,15 +46,6 @@ type (
 	}
 )
 
-func isInternalFunc(fiName string) bool {
-	internal := []string{`accumulateWordCount.json`, `countPlusOne.json`, `echo.json`, `internal.json`, "windows.json", "image.json", "geohash.json"}
-	for _, v := range internal {
-		if v == fiName {
-			return true
-		}
-	}
-	return false
-}
 func newUiFuncs(fi *fileFuncs) *uiFuncs {
 	if nil == fi {
 		return nil
@@ -74,7 +65,7 @@ func newUiFuncs(fi *fileFuncs) *uiFuncs {
 
 var gFuncmetadata map[string]*uiFuncs
 
-func (m *Manager) readFuncMetaDir() error {
+func ReadFuncMetaDir(checker InstallChecker) error {
 	gFuncmetadata = make(map[string]*uiFuncs)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
@@ -92,21 +83,22 @@ func (m *Manager) readFuncMetaDir() error {
 			continue
 		}
 
-		if err := m.readFuncMetaFile(path.Join(dir, fname)); nil != err {
+		if err := ReadFuncMetaFile(path.Join(dir, fname), checker(strings.TrimSuffix(fname, ".json"))); nil != err {
 			return err
 		}
 	}
 	return nil
 }
 
-func (m *Manager) uninstalFunc(name string) {
+func UninstallFunc(name string) {
 	if ui, ok := gFuncmetadata[name+".json"]; ok {
 		if nil != ui.About {
 			ui.About.Installed = false
 		}
 	}
 }
-func (m *Manager) readFuncMetaFile(filePath string) error {
+
+func ReadFuncMetaFile(filePath string, installed bool) error {
 	fiName := path.Base(filePath)
 	fis := new(fileFuncs)
 	err := filex.ReadJsonUnmarshal(filePath, fis)
@@ -115,15 +107,14 @@ func (m *Manager) readFuncMetaFile(filePath string) error {
 	}
 	if nil == fis.About {
 		return fmt.Errorf("not found about of %s", filePath)
-	} else if isInternalFunc(fiName) {
-		fis.About.Installed = true
 	} else {
-		_, fis.About.Installed = m.registry.Get(FUNCTION, strings.TrimSuffix(fiName, `.json`))
+		fis.About.Installed = installed
 	}
 	gFuncmetadata[fiName] = newUiFuncs(fis)
 	conf.Log.Infof("funcMeta file : %s", fiName)
 	return nil
 }
+
 func GetFunctions() (ret []*uiFuncs) {
 	for _, v := range gFuncmetadata {
 		ret = append(ret, v)

internal/meta/install_cheker.go

@@ -0,0 +1,17 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package meta
+
+type InstallChecker func(name string) bool

internal/plugin/msgUtil.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	kconf "github.com/lf-edge/ekuiper/internal/conf"
@@ -33,7 +33,7 @@ func getMsg(language, section, key string) string {
 	}
 	return ""
 }
-func (m *Manager) readUiMsgDir() error {
+func ReadUiMsgDir() error {
 	gUimsg = make(map[string]*ini.File)
 	confDir, err := kconf.GetConfLoc()
 	if nil != err {

internal/plugin/sinkMeta.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	"fmt"
@@ -92,15 +92,6 @@ type (
 	}
 )
 
-func isInternalSink(fiName string) bool {
-	internal := []string{`edgex.json`, `log.json`, `mqtt.json`, `nop.json`, `rest.json`}
-	for _, v := range internal {
-		if v == fiName {
-			return true
-		}
-	}
-	return false
-}
 func newLanguage(fi *fileLanguage) *language {
 	if nil == fi {
 		return nil
@@ -165,7 +156,8 @@ func newUiSink(fi *fileSink) (*uiSink, error) {
 }
 
 var gSinkmetadata map[string]*uiSink //immutable
-func (m *Manager) readSinkMetaDir() error {
+
+func ReadSinkMetaDir(checker InstallChecker) error {
 	gSinkmetadata = make(map[string]*uiSink)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
@@ -184,23 +176,22 @@ func (m *Manager) readSinkMetaDir() error {
 		}
 
 		filePath := path.Join(dir, fname)
-		if err := m.readSinkMetaFile(filePath); nil != err {
+		if err := ReadSinkMetaFile(filePath, checker(strings.TrimSuffix(fname, ".json"))); nil != err {
 			return err
 		}
 	}
 	return nil
 }
 
-func (m *Manager) uninstalSink(name string) {
+func UninstallSink(name string) {
 	if ui, ok := gSinkmetadata[name+".json"]; ok {
 		if nil != ui.About {
 			ui.About.Installed = false
 		}
 	}
 }
-func (m *Manager) readSinkMetaFile(filePath string) error {
+func ReadSinkMetaFile(filePath string, installed bool) error {
 	finame := path.Base(filePath)
-	pluginName := strings.TrimSuffix(finame, `.json`)
 	metadata := new(fileSink)
 	err := filex.ReadJsonUnmarshal(filePath, metadata)
 	if nil != err {
@@ -208,10 +199,8 @@ func (m *Manager) readSinkMetaFile(filePath string) error {
 	}
 	if nil == metadata.About {
 		return fmt.Errorf("not found about of %s", finame)
-	} else if isInternalSink(finame) {
-		metadata.About.Installed = true
 	} else {
-		_, metadata.About.Installed = m.registry.Get(SINK, pluginName)
+		metadata.About.Installed = installed
 	}
 	gSinkmetadata[finame], err = newUiSink(metadata)
 	if nil != err {

internal/plugin/sinkMeta_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	"testing"

internal/plugin/sourceMeta.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	"encoding/json"
@@ -43,15 +43,6 @@ type (
 	}
 )
 
-func isInternalSource(fiName string) bool {
-	internal := []string{`edgex.json`, `httppull.json`, `mqtt.json`}
-	for _, v := range internal {
-		if v == fiName {
-			return true
-		}
-	}
-	return false
-}
 func newUiSource(fi *fileSource) (*uiSource, error) {
 	if nil == fi {
 		return nil, nil
@@ -73,7 +64,7 @@ func newUiSource(fi *fileSource) (*uiSource, error) {
 
 var gSourceproperty map[string]*sourceProperty
 
-func (m *Manager) uninstalSource(name string) {
+func UninstallSource(name string) {
 	if v, ok := gSourceproperty[name+".json"]; ok {
 		if ui := v.meta; nil != ui {
 			if nil != ui.About {
@@ -82,7 +73,7 @@ func (m *Manager) uninstalSource(name string) {
 		}
 	}
 }
-func (m *Manager) readSourceMetaFile(filePath string) error {
+func ReadSourceMetaFile(filePath string, installed bool) error {
 	fileName := path.Base(filePath)
 	if "mqtt_source.json" == fileName {
 		fileName = "mqtt.json"
@@ -97,10 +88,8 @@ func (m *Manager) readSourceMetaFile(filePath string) error {
 	}
 	if nil == ptrMeta.About {
 		return fmt.Errorf("not found about of %s", filePath)
-	} else if isInternalSource(fileName) {
-		ptrMeta.About.Installed = true
 	} else {
-		_, ptrMeta.About.Installed = m.registry.Get(SOURCE, strings.TrimSuffix(fileName, `.json`))
+		ptrMeta.About.Installed = installed
 	}
 
 	yamlData := make(map[string]map[string]interface{})
@@ -124,7 +113,7 @@ func (m *Manager) readSourceMetaFile(filePath string) error {
 	return err
 }
 
-func (m *Manager) readSourceMetaDir() error {
+func ReadSourceMetaDir(checker InstallChecker) error {
 	gSourceproperty = make(map[string]*sourceProperty)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
@@ -137,7 +126,7 @@ func (m *Manager) readSourceMetaDir() error {
 		return err
 	}
 
-	if err = m.readSourceMetaFile(path.Join(confDir, "mqtt_source.json")); nil != err {
+	if err = ReadSourceMetaFile(path.Join(confDir, "mqtt_source.json"), false); nil != err {
 		return err
 	}
 
@@ -145,7 +134,7 @@ func (m *Manager) readSourceMetaDir() error {
 		fileName := info.Name()
 		if strings.HasSuffix(fileName, ".json") {
 			filePath := path.Join(dir, fileName)
-			if err = m.readSourceMetaFile(filePath); nil != err {
+			if err = ReadSourceMetaFile(filePath, checker(strings.TrimSuffix(fileName, ".json"))); nil != err {
 				return err
 			}
 			conf.Log.Infof("sourceMeta file : %s", fileName)

internal/plugin/sourceMeta_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package meta
 
 import (
 	"encoding/json"

internal/plugin/manager.go

@@ -12,20 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+// Manage the loading of both native and portable plugins
+
+package native
 
 import (
 	"archive/zip"
 	"bytes"
-	"errors"
 	"fmt"
 	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/meta"
 	"github.com/lf-edge/ekuiper/internal/pkg/filex"
 	"github.com/lf-edge/ekuiper/internal/pkg/httpx"
 	"github.com/lf-edge/ekuiper/internal/pkg/store"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/errorx"
 	"github.com/lf-edge/ekuiper/pkg/kv"
+	"github.com/pkg/errors"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -39,91 +42,102 @@ import (
 	"unicode"
 )
 
-type Plugin interface {
-	GetName() string
-	GetFile() string
-	GetShellParas() []string
-	GetSymbols() []string
-	SetName(n string)
-}
-
-type IOPlugin struct {
-	Name       string   `json:"name"`
-	File       string   `json:"file"`
-	ShellParas []string `json:"shellParas"`
-}
-
-func (p *IOPlugin) GetName() string {
-	return p.Name
-}
+// Manager Initialized in the binder
+var manager *Manager
 
-func (p *IOPlugin) GetFile() string {
-	return p.File
-}
-
-func (p *IOPlugin) GetShellParas() []string {
-	return p.ShellParas
-}
-
-func (p *IOPlugin) GetSymbols() []string {
-	return nil
-}
+const DELETED = "$deleted"
 
-func (p *IOPlugin) SetName(n string) {
-	p.Name = n
+//Manager is append only because plugin cannot delete or reload. To delete a plugin, restart the server to reindex
+type Manager struct {
+	sync.RWMutex
+	// 3 maps for source/sink/function. In each map, key is the plugin name, value is the version
+	plugins []map[string]string
+	// A map from function name to its plugin file name. It is constructed during initialization by reading kv info. All functions must have at least an entry, even the function resizes in a one function plugin.
+	symbols map[string]string
+	// loaded symbols in current runtime
+	runtime map[string]plugin.Symbol
+	// dirs
+	pluginDir string
+	etcDir    string
+	// the access to db
+	db kv.KeyValue
 }
 
-type FuncPlugin struct {
-	IOPlugin
-	// Optional, if not specified, a default element with the same name of the file will be registered
-	Functions []string `json:"functions"`
-}
+// InitManager must only be called once
+func InitManager() (*Manager, error) {
+	pluginDir, err := conf.GetPluginsLoc()
+	if err != nil {
+		return nil, fmt.Errorf("cannot find plugins folder: %s", err)
+	}
+	etcDir, err := conf.GetConfLoc()
+	if err != nil {
+		return nil, fmt.Errorf("cannot find etc folder: %s", err)
+	}
+	err, db := store.GetKV("pluginFuncs")
+	if err != nil {
+		return nil, fmt.Errorf("error when opening db: %v", err)
+	}
+	plugins := make([]map[string]string, 3)
+	for i := range PluginTypes {
+		names, err := findAll(PluginType(i), pluginDir)
+		if err != nil {
+			return nil, fmt.Errorf("fail to find existing plugins: %s", err)
+		}
+		plugins[i] = names
+	}
+	registry := &Manager{plugins: plugins, symbols: make(map[string]string), db: db, pluginDir: pluginDir, etcDir: etcDir, runtime: make(map[string]plugin.Symbol)}
 
-func (fp *FuncPlugin) GetSymbols() []string {
-	return fp.Functions
+	for pf := range plugins[FUNCTION] {
+		l := make([]string, 0)
+		if ok, err := db.Get(pf, &l); ok {
+			registry.storeSymbols(pf, l)
+		} else if err != nil {
+			return nil, fmt.Errorf("error when querying kv: %s", err)
+		} else {
+			registry.storeSymbols(pf, []string{pf})
+		}
+	}
+	manager = registry
+	return registry, nil
 }
 
-type PluginType int
+func findAll(t PluginType, pluginDir string) (result map[string]string, err error) {
+	result = make(map[string]string)
+	dir := path.Join(pluginDir, PluginTypes[t])
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return
+	}
 
-func NewPluginByType(t PluginType) Plugin {
-	switch t {
-	case FUNCTION:
-		return &FuncPlugin{}
-	default:
-		return &IOPlugin{}
+	for _, file := range files {
+		baseName := filepath.Base(file.Name())
+		if strings.HasSuffix(baseName, ".so") {
+			n, v := parseName(baseName)
+			result[n] = v
+		}
 	}
+	return
 }
 
-const (
-	SOURCE PluginType = iota
-	SINK
-	FUNCTION
-)
-
-const DELETED = "$deleted"
-
-var (
-	PluginTypes = []string{"sources", "sinks", "functions"}
-	once        sync.Once
-	singleton   *Manager
-)
+func GetManager() *Manager {
+	return manager
+}
 
-//Registry is append only because plugin cannot delete or reload. To delete a plugin, restart the server to reindex
-type Registry struct {
-	sync.RWMutex
-	// 3 maps for source/sink/function. In each map, key is the plugin name, value is the version
-	plugins []map[string]string
-	// A map from function name to its plugin file name. It is constructed during initialization by reading kv info. All functions must have at least an entry, even the function resizes in a one function plugin.
-	symbols map[string]string
+func (rr *Manager) get(t PluginType, name string) (string, bool) {
+	rr.RLock()
+	result := rr.plugins[t]
+	rr.RUnlock()
+	r, ok := result[name]
+	return r, ok
 }
 
-func (rr *Registry) Store(t PluginType, name string, version string) {
+func (rr *Manager) store(t PluginType, name string, version string) {
 	rr.Lock()
 	rr.plugins[t][name] = version
 	rr.Unlock()
 }
 
-func (rr *Registry) StoreSymbols(name string, symbols []string) error {
+func (rr *Manager) storeSymbols(name string, symbols []string) error {
 	rr.Lock()
 	defer rr.Unlock()
 	for _, s := range symbols {
@@ -137,7 +151,7 @@ func (rr *Registry) StoreSymbols(name string, symbols []string) error {
 	return nil
 }
 
-func (rr *Registry) RemoveSymbols(symbols []string) {
+func (rr *Manager) removeSymbols(symbols []string) {
 	rr.Lock()
 	for _, s := range symbols {
 		delete(rr.symbols, s)
@@ -145,7 +159,9 @@ func (rr *Registry) RemoveSymbols(symbols []string) {
 	rr.Unlock()
 }
 
-func (rr *Registry) List(t PluginType) []string {
+// API for management
+
+func (rr *Manager) List(t PluginType) []string {
 	rr.RLock()
 	result := rr.plugins[t]
 	rr.RUnlock()
@@ -156,7 +172,7 @@ func (rr *Registry) List(t PluginType) []string {
 	return keys
 }
 
-func (rr *Registry) ListSymbols() []string {
+func (rr *Manager) ListSymbols() []string {
 	rr.RLock()
 	result := rr.symbols
 	rr.RUnlock()
@@ -167,15 +183,7 @@ func (rr *Registry) ListSymbols() []string {
 	return keys
 }
 
-func (rr *Registry) Get(t PluginType, name string) (string, bool) {
-	rr.RLock()
-	result := rr.plugins[t]
-	rr.RUnlock()
-	r, ok := result[name]
-	return r, ok
-}
-
-func (rr *Registry) GetPluginVersionBySymbol(t PluginType, symbolName string) (string, bool) {
+func (rr *Manager) GetPluginVersionBySymbol(t PluginType, symbolName string) (string, bool) {
 	switch t {
 	case FUNCTION:
 		rr.RLock()
@@ -189,11 +197,11 @@ func (rr *Registry) GetPluginVersionBySymbol(t PluginType, symbolName string) (s
 			return "", false
 		}
 	default:
-		return rr.Get(t, symbolName)
+		return rr.get(t, symbolName)
 	}
 }
 
-func (rr *Registry) GetPluginBySymbol(t PluginType, symbolName string) (string, bool) {
+func (rr *Manager) GetPluginBySymbol(t PluginType, symbolName string) (string, bool) {
 	switch t {
 	case FUNCTION:
 		rr.RLock()
@@ -205,175 +213,7 @@ func (rr *Registry) GetPluginBySymbol(t PluginType, symbolName string) (string,
 	}
 }
 
-var symbolRegistry = make(map[string]plugin.Symbol)
-var mu sync.RWMutex
-
-func getPlugin(t string, pt PluginType) (plugin.Symbol, error) {
-	ut := ucFirst(t)
-	ptype := PluginTypes[pt]
-	key := ptype + "/" + t
-	mu.Lock()
-	defer mu.Unlock()
-	var nf plugin.Symbol
-	nf, ok := symbolRegistry[key]
-	if !ok {
-		m, err := NewPluginManager()
-		if err != nil {
-			return nil, fmt.Errorf("fail to initialize the plugin manager")
-		}
-		mod, err := getSoFilePath(m, pt, t, false)
-		if err != nil {
-			return nil, fmt.Errorf("cannot get the plugin file path: %v", err)
-		}
-		conf.Log.Debugf("Opening plugin %s", mod)
-		plug, err := plugin.Open(mod)
-		if err != nil {
-			return nil, fmt.Errorf("cannot open %s: %v", mod, err)
-		}
-		conf.Log.Debugf("Successfully open plugin %s", mod)
-		nf, err = plug.Lookup(ut)
-		if err != nil {
-			return nil, fmt.Errorf("cannot find symbol %s, please check if it is exported", t)
-		}
-		conf.Log.Debugf("Successfully look-up plugin %s", mod)
-		symbolRegistry[key] = nf
-	}
-	return nf, nil
-}
-
-func GetSource(t string) (api.Source, error) {
-	nf, err := getPlugin(t, SOURCE)
-	if err != nil {
-		return nil, err
-	}
-	var s api.Source
-	switch t := nf.(type) {
-	case api.Source:
-		s = t
-	case func() api.Source:
-		s = t()
-	default:
-		return nil, fmt.Errorf("exported symbol %s is not type of api.Source or function that return api.Source", t)
-	}
-	return s, nil
-}
-
-func GetSink(t string) (api.Sink, error) {
-	nf, err := getPlugin(t, SINK)
-	if err != nil {
-		return nil, err
-	}
-	var s api.Sink
-	switch t := nf.(type) {
-	case api.Sink:
-		s = t
-	case func() api.Sink:
-		s = t()
-	default:
-		return nil, fmt.Errorf("exported symbol %s is not type of api.Sink or function that return api.Sink", t)
-	}
-	return s, nil
-}
-
-type Manager struct {
-	pluginDir string
-	etcDir    string
-	registry  *Registry
-	db        kv.KeyValue
-}
-
-func NewPluginManager() (*Manager, error) {
-	var outerErr error
-	once.Do(func() {
-		dir, err := conf.GetPluginsLoc()
-		if err != nil {
-			outerErr = fmt.Errorf("cannot find plugins folder: %s", err)
-			return
-		}
-		etcDir, err := conf.GetConfLoc()
-		if err != nil {
-			outerErr = fmt.Errorf("cannot find etc folder: %s", err)
-			return
-		}
-		err, db := store.GetKV("pluginFuncs")
-		if err != nil {
-			outerErr = fmt.Errorf("error when opening db: %v.", err)
-		}
-		plugins := make([]map[string]string, 3)
-		for i := 0; i < 3; i++ {
-			names, err := findAll(PluginType(i), dir)
-			if err != nil {
-				outerErr = fmt.Errorf("fail to find existing plugins: %s", err)
-				return
-			}
-			plugins[i] = names
-		}
-		registry := &Registry{plugins: plugins, symbols: make(map[string]string)}
-		for pf := range plugins[FUNCTION] {
-			l := make([]string, 0)
-			if ok, err := db.Get(pf, &l); ok {
-				registry.StoreSymbols(pf, l)
-			} else if err != nil {
-				outerErr = fmt.Errorf("error when querying kv: %s", err)
-				return
-			} else {
-				registry.StoreSymbols(pf, []string{pf})
-			}
-		}
-
-		singleton = &Manager{
-			pluginDir: dir,
-			etcDir:    etcDir,
-			registry:  registry,
-			db:        db,
-		}
-		if err := singleton.readSourceMetaDir(); nil != err {
-			conf.Log.Errorf("readSourceMetaDir:%v", err)
-		}
-		if err := singleton.readSinkMetaDir(); nil != err {
-			conf.Log.Errorf("readSinkMetaDir:%v", err)
-		}
-		if err := singleton.readFuncMetaDir(); nil != err {
-			conf.Log.Errorf("readFuncMetaDir:%v", err)
-		}
-		if err := singleton.readUiMsgDir(); nil != err {
-			conf.Log.Errorf("readUiMsgDir:%v", err)
-		}
-	})
-	return singleton, outerErr
-}
-
-func findAll(t PluginType, pluginDir string) (result map[string]string, err error) {
-	result = make(map[string]string)
-	dir := path.Join(pluginDir, PluginTypes[t])
-	files, err := ioutil.ReadDir(dir)
-	if err != nil {
-		return
-	}
-
-	for _, file := range files {
-		baseName := filepath.Base(file.Name())
-		if strings.HasSuffix(baseName, ".so") {
-			n, v := parseName(baseName)
-			result[n] = v
-		}
-	}
-	return
-}
-
-func (m *Manager) List(t PluginType) (result []string, err error) {
-	return m.registry.List(t), nil
-}
-
-func (m *Manager) ListSymbols() (result []string, err error) {
-	return m.registry.ListSymbols(), nil
-}
-
-func (m *Manager) GetSymbol(s string) (result string, ok bool) {
-	return m.registry.GetPluginBySymbol(FUNCTION, s)
-}
-
-func (m *Manager) Register(t PluginType, j Plugin) error {
+func (rr *Manager) Register(t PluginType, j Plugin) error {
 	name, uri, shellParas := j.GetName(), j.GetFile(), j.GetShellParas()
 	//Validation
 	name = strings.Trim(name, " ")
@@ -384,7 +224,7 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 		return fmt.Errorf("invalid uri %s", uri)
 	}
 
-	if v, ok := m.registry.Get(t, name); ok {
+	if v, ok := rr.get(t, name); ok {
 		if v == DELETED {
 			return fmt.Errorf("invalid name %s: the plugin is marked as deleted but Kuiper is not restarted for the change to take effect yet", name)
 		} else {
@@ -394,20 +234,20 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 	var err error
 	if t == FUNCTION {
 		if len(j.GetSymbols()) > 0 {
-			err = m.db.Set(name, j.GetSymbols())
+			err = rr.db.Set(name, j.GetSymbols())
 			if err != nil {
 				return err
 			}
-			err = m.registry.StoreSymbols(name, j.GetSymbols())
+			err = rr.storeSymbols(name, j.GetSymbols())
 		} else {
-			err = m.registry.StoreSymbols(name, []string{name})
+			err = rr.storeSymbols(name, []string{name})
 		}
 	}
 	if err != nil {
 		return err
 	}
 
-	zipPath := path.Join(m.pluginDir, name+".zip")
+	zipPath := path.Join(rr.pluginDir, name+".zip")
 	var unzipFiles []string
 	//clean up: delete zip file and unzip files in error
 	defer os.Remove(zipPath)
@@ -417,66 +257,66 @@ func (m *Manager) Register(t PluginType, j Plugin) error {
 		return fmt.Errorf("fail to download file %s: %s", uri, err)
 	}
 	//unzip and copy to destination
-	unzipFiles, version, err := m.install(t, name, zipPath, shellParas)
+	unzipFiles, version, err := rr.install(t, name, zipPath, shellParas)
 	if err == nil && len(j.GetSymbols()) > 0 {
-		err = m.db.Set(name, j.GetSymbols())
+		err = rr.db.Set(name, j.GetSymbols())
 	}
 	if err != nil { //Revert for any errors
 		if t == SOURCE && len(unzipFiles) == 1 { //source that only copy so file
 			os.RemoveAll(unzipFiles[0])
 		}
 		if len(j.GetSymbols()) > 0 {
-			m.registry.RemoveSymbols(j.GetSymbols())
+			rr.removeSymbols(j.GetSymbols())
 		} else {
-			m.registry.RemoveSymbols([]string{name})
+			rr.removeSymbols([]string{name})
 		}
 		return fmt.Errorf("fail to install plugin: %s", err)
 	}
-	m.registry.Store(t, name, version)
+	rr.store(t, name, version)
 
 	switch t {
 	case SINK:
-		if err := m.readSinkMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
+		if err := meta.ReadSinkMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSinkFile:%v", err)
 		}
 	case SOURCE:
-		if err := m.readSourceMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
+		if err := meta.ReadSourceMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSourceFile:%v", err)
 		}
 	case FUNCTION:
-		if err := m.readFuncMetaFile(path.Join(m.etcDir, PluginTypes[t], name+`.json`)); nil != err {
+		if err := meta.ReadFuncMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readFuncFile:%v", err)
 		}
 	}
 	return nil
 }
 
-// prerequisite：function plugin of name exists
-func (m *Manager) RegisterFuncs(name string, functions []string) error {
+// RegisterFuncs prerequisite：function plugin of name exists
+func (rr *Manager) RegisterFuncs(name string, functions []string) error {
 	if len(functions) == 0 {
 		return fmt.Errorf("property 'functions' must not be empty")
 	}
 	old := make([]string, 0)
-	if ok, err := m.db.Get(name, &old); err != nil {
+	if ok, err := rr.db.Get(name, &old); err != nil {
 		return err
 	} else if ok {
-		m.registry.RemoveSymbols(old)
+		rr.removeSymbols(old)
 	} else if !ok {
-		m.registry.RemoveSymbols([]string{name})
+		rr.removeSymbols([]string{name})
 	}
-	err := m.db.Set(name, functions)
+	err := rr.db.Set(name, functions)
 	if err != nil {
 		return err
 	}
-	return m.registry.StoreSymbols(name, functions)
+	return rr.storeSymbols(name, functions)
 }
 
-func (m *Manager) Delete(t PluginType, name string, stop bool) error {
+func (rr *Manager) Delete(t PluginType, name string, stop bool) error {
 	name = strings.Trim(name, " ")
 	if name == "" {
 		return fmt.Errorf("invalid name %s: should not be empty", name)
 	}
-	soPath, err := getSoFilePath(m, t, name, true)
+	soPath, err := rr.getSoFilePath(t, name, true)
 	if err != nil {
 		return err
 	}
@@ -485,7 +325,7 @@ func (m *Manager) Delete(t PluginType, name string, stop bool) error {
 		soPath,
 	}
 	// Find etc folder
-	etcPath := path.Join(m.etcDir, PluginTypes[t], name)
+	etcPath := path.Join(rr.etcDir, PluginTypes[t], name)
 	if fi, err := os.Stat(etcPath); err == nil {
 		if fi.Mode().IsDir() {
 			paths = append(paths, etcPath)
@@ -493,24 +333,24 @@ func (m *Manager) Delete(t PluginType, name string, stop bool) error {
 	}
 	switch t {
 	case SOURCE:
-		paths = append(paths, path.Join(m.etcDir, PluginTypes[t], name+".yaml"))
-		m.uninstalSource(name)
+		paths = append(paths, path.Join(rr.etcDir, PluginTypes[t], name+".yaml"))
+		meta.UninstallSource(name)
 	case SINK:
-		m.uninstalSink(name)
+		meta.UninstallSink(name)
 	case FUNCTION:
 		old := make([]string, 0)
-		if ok, err := m.db.Get(name, &old); err != nil {
+		if ok, err := rr.db.Get(name, &old); err != nil {
 			return err
 		} else if ok {
-			m.registry.RemoveSymbols(old)
-			err := m.db.Delete(name)
+			rr.removeSymbols(old)
+			err := rr.db.Delete(name)
 			if err != nil {
 				return err
 			}
 		} else if !ok {
-			m.registry.RemoveSymbols([]string{name})
+			rr.removeSymbols([]string{name})
 		}
-		m.uninstalFunc(name)
+		meta.UninstallFunc(name)
 	}
 
 	for _, p := range paths {
@@ -528,7 +368,7 @@ func (m *Manager) Delete(t PluginType, name string, stop bool) error {
 	if len(results) > 0 {
 		return errors.New(strings.Join(results, "\n"))
 	} else {
-		m.registry.Store(t, name, DELETED)
+		rr.store(t, name, DELETED)
 		if stop {
 			go func() {
 				time.Sleep(1 * time.Second)
@@ -538,8 +378,8 @@ func (m *Manager) Delete(t PluginType, name string, stop bool) error {
 		return nil
 	}
 }
-func (m *Manager) Get(t PluginType, name string) (map[string]interface{}, bool) {
-	v, ok := m.registry.Get(t, name)
+func (rr *Manager) GetPluginInfo(t PluginType, name string) (map[string]interface{}, bool) {
+	v, ok := rr.get(t, name)
 	if strings.HasPrefix(v, "v") {
 		v = v[1:]
 	}
@@ -550,7 +390,7 @@ func (m *Manager) Get(t PluginType, name string) (map[string]interface{}, bool)
 		}
 		if t == FUNCTION {
 			l := make([]string, 0)
-			if ok, _ := m.db.Get(name, &l); ok {
+			if ok, _ := rr.db.Get(name, &l); ok {
 				r["functions"] = l
 			}
 			// ignore the error
@@ -560,70 +400,9 @@ func (m *Manager) Get(t PluginType, name string) (map[string]interface{}, bool)
 	return nil, false
 }
 
-// Start implement xsql.FunctionRegister
-
-func (m *Manager) HasFunction(name string) bool {
-	_, ok := m.GetSymbol(name)
-	return ok
-}
-
-func (m *Manager) Function(name string) (api.Function, error) {
-	nf, err := getPlugin(name, FUNCTION)
-	if err != nil {
-		return nil, err
-	}
-	var s api.Function
-	switch t := nf.(type) {
-	case api.Function:
-		s = t
-	case func() api.Function:
-		s = t()
-	default:
-		return nil, fmt.Errorf("exported symbol %s is not type of api.Function or function that return api.Function", t)
-	}
-	return s, nil
-}
-
-// End Implement FunctionRegister
-
-// Return the lowercase version of so name. It may be upper case in path.
-func getSoFilePath(m *Manager, t PluginType, name string, isSoName bool) (string, error) {
-	var (
-		v      string
-		soname string
-		ok     bool
-	)
-	// We must identify plugin or symbol when deleting function plugin
-	if isSoName {
-		soname = name
-	} else {
-		soname, ok = m.registry.GetPluginBySymbol(t, name)
-		if !ok {
-			return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid symbol name %s: not exist", name))
-		}
-	}
-	v, ok = m.registry.Get(t, soname)
-	if !ok {
-		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid name %s: not exist", soname))
-	}
-
-	soFile := soname + ".so"
-	if v != "" {
-		soFile = fmt.Sprintf("%s@%s.so", soname, v)
-	}
-	p := path.Join(m.pluginDir, PluginTypes[t], soFile)
-	if _, err := os.Stat(p); err != nil {
-		p = path.Join(m.pluginDir, PluginTypes[t], ucFirst(soFile))
-	}
-	if _, err := os.Stat(p); err != nil {
-		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("cannot find .so file for plugin %s", soname))
-	}
-	return p, nil
-}
-
-func (m *Manager) install(t PluginType, name, src string, shellParas []string) ([]string, string, error) {
+func (rr *Manager) install(t PluginType, name, src string, shellParas []string) ([]string, string, error) {
 	var filenames []string
-	var tempPath = path.Join(m.pluginDir, "temp", PluginTypes[t], name)
+	var tempPath = path.Join(rr.pluginDir, "temp", PluginTypes[t], name)
 	defer os.RemoveAll(tempPath)
 	r, err := zip.OpenReader(src)
 	if err != nil {
@@ -636,7 +415,7 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 	expFiles := 1
 	if t == SOURCE {
 		yamlFile = name + ".yaml"
-		yamlPath = path.Join(m.etcDir, PluginTypes[t], yamlFile)
+		yamlPath = path.Join(rr.etcDir, PluginTypes[t], yamlFile)
 		expFiles = 2
 	}
 	var revokeFiles []string
@@ -651,14 +430,14 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 			revokeFiles = append(revokeFiles, yamlPath)
 			filenames = append(filenames, yamlPath)
 		} else if fileName == name+".json" {
-			jsonPath := path.Join(m.etcDir, PluginTypes[t], fileName)
+			jsonPath := path.Join(rr.etcDir, PluginTypes[t], fileName)
 			if err := filex.UnzipTo(file, jsonPath); nil != err {
 				conf.Log.Errorf("Failed to decompress the metadata %s file", fileName)
 			} else {
 				revokeFiles = append(revokeFiles, jsonPath)
 			}
 		} else if soPrefix.Match([]byte(fileName)) {
-			soPath := path.Join(m.pluginDir, PluginTypes[t], fileName)
+			soPath := path.Join(rr.pluginDir, PluginTypes[t], fileName)
 			err = filex.UnzipTo(file, soPath)
 			if err != nil {
 				return filenames, "", err
@@ -667,7 +446,7 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 			revokeFiles = append(revokeFiles, soPath)
 			_, version = parseName(fileName)
 		} else if strings.HasPrefix(fileName, "etc/") {
-			err = filex.UnzipTo(file, path.Join(m.etcDir, PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
+			err = filex.UnzipTo(file, path.Join(rr.etcDir, PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
 			if err != nil {
 				return filenames, "", err
 			}
@@ -711,6 +490,140 @@ func (m *Manager) install(t PluginType, name, src string, shellParas []string) (
 	return filenames, version, nil
 }
 
+// binder factory implementations
+
+func (rr *Manager) Source(name string) (api.Source, error) {
+	nf, err := rr.loadRuntime(SOURCE, name)
+	if err != nil {
+		return nil, err
+	}
+	if nf == nil {
+		return nil, nil
+	}
+	switch t := nf.(type) {
+	case api.Source:
+		return t, nil
+	case func() api.Source:
+		return t(), nil
+	default:
+		return nil, fmt.Errorf("exported symbol %s is not type of api.Source or function that return api.Source", t)
+	}
+}
+
+func (rr *Manager) Sink(name string) (api.Sink, error) {
+	nf, err := rr.loadRuntime(SINK, name)
+	if err != nil {
+		return nil, err
+	}
+	if nf == nil {
+		return nil, nil
+	}
+	var s api.Sink
+	switch t := nf.(type) {
+	case api.Sink:
+		s = t
+	case func() api.Sink:
+		s = t()
+	default:
+		return nil, fmt.Errorf("exported symbol %s is not type of api.Sink or function that return api.Sink", t)
+	}
+	return s, nil
+}
+
+func (rr *Manager) Function(name string) (api.Function, error) {
+	nf, err := rr.loadRuntime(FUNCTION, name)
+	if err != nil {
+		return nil, err
+	}
+	if nf == nil {
+		return nil, nil
+	}
+	var s api.Function
+	switch t := nf.(type) {
+	case api.Function:
+		s = t
+	case func() api.Function:
+		s = t()
+	default:
+		return nil, fmt.Errorf("exported symbol %s is not type of api.Function or function that return api.Function", t)
+	}
+	return s, nil
+}
+
+func (rr *Manager) HasFunctionSet(name string) bool {
+	_, ok := rr.get(FUNCTION, name)
+	return ok
+}
+
+// If not found, return nil,nil; Other errors return nil, err
+func (rr *Manager) loadRuntime(t PluginType, name string) (plugin.Symbol, error) {
+	ut := ucFirst(name)
+	ptype := PluginTypes[t]
+	key := ptype + "/" + name
+	var nf plugin.Symbol
+	rr.RLock()
+	nf, ok := rr.runtime[key]
+	rr.RUnlock()
+	if !ok {
+		mod, err := rr.getSoFilePath(t, name, false)
+		if err != nil {
+			conf.Log.Debugf(fmt.Sprintf("cannot find the native plugin in path: %v", err))
+			return nil, nil
+		}
+		conf.Log.Debugf("Opening plugin %s", mod)
+		plug, err := plugin.Open(mod)
+		if err != nil {
+			return nil, fmt.Errorf("cannot open %s: %v", mod, err)
+		}
+		conf.Log.Debugf("Successfully open plugin %s", mod)
+		nf, err = plug.Lookup(ut)
+		if err != nil {
+			conf.Log.Debugf(fmt.Sprintf("cannot find symbol %s, please check if it is exported", name))
+			return nil, nil
+		}
+		conf.Log.Debugf("Successfully look-up plugin %s", mod)
+		rr.Lock()
+		rr.runtime[key] = nf
+		rr.Unlock()
+	}
+	return nf, nil
+}
+
+// Return the lowercase version of so name. It may be upper case in path.
+func (rr *Manager) getSoFilePath(t PluginType, name string, isSoName bool) (string, error) {
+	var (
+		v      string
+		soname string
+		ok     bool
+	)
+	// We must identify plugin or symbol when deleting function plugin
+	if isSoName {
+		soname = name
+	} else {
+		soname, ok = rr.GetPluginBySymbol(t, name)
+		if !ok {
+			return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid symbol name %s: not exist", name))
+		}
+	}
+	v, ok = rr.get(t, soname)
+	if !ok {
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("invalid name %s: not exist", soname))
+	}
+
+	soFile := soname + ".so"
+	if v != "" {
+		soFile = fmt.Sprintf("%s@%s.so", soname, v)
+	}
+	p := path.Join(rr.pluginDir, PluginTypes[t], soFile)
+	if _, err := os.Stat(p); err != nil {
+		p = path.Join(rr.pluginDir, PluginTypes[t], ucFirst(soFile))
+	}
+	if _, err := os.Stat(p); err != nil {
+		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("cannot find .so file for plugin %s", soname))
+	}
+	return p, nil
+}
+
 func parseName(n string) (string, string) {
 	result := strings.Split(n, ".so")
 	result = strings.Split(result[0], "@")

internal/plugin/manager_test.go

@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package plugin
+package native
 
 import (
 	"errors"
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
 	"github.com/lf-edge/ekuiper/internal/testx"
-	"github.com/lf-edge/ekuiper/internal/xsql"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -28,21 +29,21 @@ import (
 	"testing"
 )
 
-var manager *Manager
-
 func init() {
-	var err error
 	testx.InitEnv()
-	manager, err = NewPluginManager()
+	nativeManager, err := InitManager()
+	if err != nil {
+		panic(err)
+	}
+	err = function.Initialize([]binder.FactoryEntry{{Name: "native plugin", Factory: nativeManager}})
 	if err != nil {
 		panic(err)
 	}
-	xsql.InitFuncRegisters(manager)
 }
 
 func TestManager_Register(t *testing.T) {
 	s := httptest.NewServer(
-		http.FileServer(http.Dir("testzips")),
+		http.FileServer(http.Dir("../testzips")),
 	)
 	defer s.Close()
 	endpoint := s.URL
@@ -167,11 +168,7 @@ func TestManager_List(t *testing.T) {
 	fmt.Printf("The test bucket size is %d.\n\n", len(data))
 
 	for i, p := range data {
-		result, err := manager.List(p.t)
-		if err != nil {
-			t.Errorf("%d: list error : %s\n\n", i, err)
-			return
-		}
+		result := manager.List(p.t)
 		sort.Strings(result)
 		if !reflect.DeepEqual(p.r, result) {
 			t.Errorf("%d: result mismatch:\n  exp=%v\n  got=%v\n\n", i, p.r, result)
@@ -181,16 +178,12 @@ func TestManager_List(t *testing.T) {
 
 func TestManager_Symbols(t *testing.T) {
 	r := []string{"accumulateWordCount", "comp", "countPlusOne", "echo", "echo2", "echo3", "misc"}
-	result, err := manager.ListSymbols()
-	if err != nil {
-		t.Errorf("list symbols error : %s\n\n", err)
-		return
-	}
+	result := manager.ListSymbols()
 	sort.Strings(result)
 	if !reflect.DeepEqual(r, result) {
 		t.Errorf("result mismatch:\n  exp=%v\n  got=%v\n\n", r, result)
 	}
-	p, ok := manager.GetSymbol("echo3")
+	p, ok := manager.GetPluginBySymbol(FUNCTION, "echo3")
 	if !ok {
 		t.Errorf("cannot find echo3 symbol")
 	}
@@ -232,7 +225,7 @@ func TestManager_Desc(t *testing.T) {
 	fmt.Printf("The test bucket size is %d.\n\n", len(data))
 
 	for i, p := range data {
-		result, ok := manager.Get(p.t, p.n)
+		result, ok := manager.GetPluginInfo(p.t, p.n)
 		if !ok {
 			t.Errorf("%d: get error : not found\n\n", i)
 			return

internal/plugin/native/plugin.go

@@ -0,0 +1,79 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package native
+
+type PluginType int
+
+const (
+	SOURCE PluginType = iota
+	SINK
+	FUNCTION
+)
+
+var PluginTypes = []string{"sources", "sinks", "functions"}
+
+type Plugin interface {
+	GetName() string
+	GetFile() string
+	GetShellParas() []string
+	GetSymbols() []string
+	SetName(n string)
+}
+
+// IOPlugin Unify model. Flat all properties for each kind.
+type IOPlugin struct {
+	Name       string   `json:"name"`
+	File       string   `json:"file"`
+	ShellParas []string `json:"shellParas"`
+}
+
+func (p *IOPlugin) GetName() string {
+	return p.Name
+}
+
+func (p *IOPlugin) GetFile() string {
+	return p.File
+}
+
+func (p *IOPlugin) GetShellParas() []string {
+	return p.ShellParas
+}
+
+func (p *IOPlugin) GetSymbols() []string {
+	return nil
+}
+
+func (p *IOPlugin) SetName(n string) {
+	p.Name = n
+}
+
+func NewPluginByType(t PluginType) Plugin {
+	switch t {
+	case FUNCTION:
+		return &FuncPlugin{}
+	default:
+		return &IOPlugin{}
+	}
+}
+
+type FuncPlugin struct {
+	IOPlugin
+	// Optional, if not specified, a default element with the same name of the file will be registered
+	Functions []string `json:"functions"`
+}
+
+func (fp *FuncPlugin) GetSymbols() []string {
+	return fp.Functions
+}

internal/server/rest.go

@@ -21,7 +21,8 @@ import (
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
 	"github.com/lf-edge/ekuiper/internal/conf"
-	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/meta"
+	"github.com/lf-edge/ekuiper/internal/plugin/native"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/ast"
@@ -421,50 +422,47 @@ func getTopoRuleHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(content))
 }
 
-func pluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
+func pluginsHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
+	pluginManager := native.GetManager()
 	defer r.Body.Close()
 	switch r.Method {
 	case http.MethodGet:
-		content, err := pluginManager.List(t)
-		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s plugins list command error", plugin.PluginTypes[t]), logger)
-			return
-		}
+		content := pluginManager.List(t)
 		jsonResponse(content, w, logger)
 	case http.MethodPost:
-		sd := plugin.NewPluginByType(t)
+		sd := native.NewPluginByType(t)
 		err := json.NewDecoder(r.Body).Decode(sd)
 		// Problems decoding
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", plugin.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", native.PluginTypes[t]), logger)
 			return
 		}
 		err = pluginManager.Register(t, sd)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s plugins create command error", plugin.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("%s plugins create command error", native.PluginTypes[t]), logger)
 			return
 		}
 		w.WriteHeader(http.StatusCreated)
-		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", plugin.PluginTypes[t], sd.GetName())))
+		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", native.PluginTypes[t], sd.GetName())))
 	}
 }
 
-func pluginHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
+func pluginHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
 	cb := r.URL.Query().Get("stop")
-
+	pluginManager := native.GetManager()
 	switch r.Method {
 	case http.MethodDelete:
 		r := cb == "1"
 		err := pluginManager.Delete(t, name, r)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", plugin.PluginTypes[t], name), logger)
+			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", native.PluginTypes[t], name), logger)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
-		result := fmt.Sprintf("%s plugin %s is deleted", plugin.PluginTypes[t], name)
+		result := fmt.Sprintf("%s plugin %s is deleted", native.PluginTypes[t], name)
 		if r {
 			result = fmt.Sprintf("%s and Kuiper will be stopped", result)
 		} else {
@@ -472,9 +470,9 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType)
 		}
 		w.Write([]byte(result))
 	case http.MethodGet:
-		j, ok := pluginManager.Get(t, name)
+		j, ok := pluginManager.GetPluginInfo(t, name)
 		if !ok {
-			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", plugin.PluginTypes[t], name), logger)
+			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", native.PluginTypes[t], name), logger)
 			return
 		}
 		jsonResponse(j, w, logger)
@@ -483,43 +481,41 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType)
 
 //list or create source plugin
 func sourcesHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugin.SOURCE)
+	pluginsHandler(w, r, native.SOURCE)
 }
 
 //delete a source plugin
 func sourceHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugin.SOURCE)
+	pluginHandler(w, r, native.SOURCE)
 }
 
 //list or create sink plugin
 func sinksHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugin.SINK)
+	pluginsHandler(w, r, native.SINK)
 }
 
 //delete a sink plugin
 func sinkHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugin.SINK)
+	pluginHandler(w, r, native.SINK)
 }
 
 //list or create function plugin
 func functionsHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, plugin.FUNCTION)
+	pluginsHandler(w, r, native.FUNCTION)
 }
 
 //list all user defined functions in all function plugins
 func functionsListHandler(w http.ResponseWriter, r *http.Request) {
-	content, err := pluginManager.ListSymbols()
-	if err != nil {
-		handleError(w, err, "udfs list command error", logger)
-		return
-	}
+	pluginManager := native.GetManager()
+	content := pluginManager.ListSymbols()
 	jsonResponse(content, w, logger)
 }
 
 func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	name := vars["name"]
-	j, ok := pluginManager.GetSymbol(name)
+	pluginManager := native.GetManager()
+	j, ok := pluginManager.GetPluginBySymbol(native.FUNCTION, name)
 	if !ok {
 		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
 		return
@@ -529,7 +525,7 @@ func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 
 //delete a function plugin
 func functionHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, plugin.FUNCTION)
+	pluginHandler(w, r, native.FUNCTION)
 }
 
 type functionList struct {
@@ -543,10 +539,10 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
-
-	_, ok := pluginManager.Get(plugin.FUNCTION, name)
+	pluginManager := native.GetManager()
+	_, ok := pluginManager.GetPluginInfo(native.FUNCTION, name)
 	if !ok {
-		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", plugin.PluginTypes[plugin.FUNCTION], name), logger)
+		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", native.PluginTypes[native.FUNCTION], name), logger)
 		return
 	}
 	sd := functionList{}
@@ -566,15 +562,15 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func prebuildSourcePlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugin.SOURCE)
+	prebuildPluginsHandler(w, r, native.SOURCE)
 }
 
 func prebuildSinkPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugin.SINK)
+	prebuildPluginsHandler(w, r, native.SINK)
 }
 
 func prebuildFuncsPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, plugin.FUNCTION)
+	prebuildPluginsHandler(w, r, native.FUNCTION)
 }
 
 func isOffcialDockerImage() bool {
@@ -584,7 +580,7 @@ func isOffcialDockerImage() bool {
 	return true
 }
 
-func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
+func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
 	emsg := "It's strongly recommended to install plugins at official released Debian Docker images. If you choose to proceed to install plugin, please make sure the plugin is already validated in your own build."
 	if !isOffcialDockerImage() {
 		handleError(w, fmt.Errorf(emsg), "", logger)
@@ -600,9 +596,9 @@ func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.Plu
 		if strings.Contains(prettyName, "DEBIAN") {
 			hosts := conf.Config.Basic.PluginHosts
 			ptype := "sources"
-			if t == plugin.SINK {
+			if t == native.SINK {
 				ptype = "sinks"
-			} else if t == plugin.FUNCTION {
+			} else if t == native.FUNCTION {
 				ptype = "functions"
 			}
 			if err, plugins := fetchPluginList(hosts, ptype, os, runtime.GOARCH); err != nil {
@@ -701,7 +697,7 @@ loop:
 //list sink plugin
 func sinksMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	sinks := plugin.GetSinks()
+	sinks := meta.GetSinks()
 	jsonResponse(sinks, w, logger)
 	return
 }
@@ -713,7 +709,7 @@ func newSinkMetaHandler(w http.ResponseWriter, r *http.Request) {
 	pluginName := vars["name"]
 
 	language := getLanguage(r)
-	ptrMetadata, err := plugin.GetSinkMeta(pluginName, language)
+	ptrMetadata, err := meta.GetSinkMeta(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -724,7 +720,7 @@ func newSinkMetaHandler(w http.ResponseWriter, r *http.Request) {
 //list functions
 func functionsMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	sinks := plugin.GetFunctions()
+	sinks := meta.GetFunctions()
 	jsonResponse(sinks, w, logger)
 	return
 }
@@ -732,7 +728,7 @@ func functionsMetaHandler(w http.ResponseWriter, r *http.Request) {
 //list source plugin
 func sourcesMetaHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
-	ret := plugin.GetSources()
+	ret := meta.GetSources()
 	if nil != ret {
 		jsonResponse(ret, w, logger)
 		return
@@ -745,7 +741,7 @@ func sourceMetaHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
 	language := getLanguage(r)
-	ret, err := plugin.GetSourceMeta(pluginName, language)
+	ret, err := meta.GetSourceMeta(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -762,7 +758,7 @@ func sourceConfHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
 	language := getLanguage(r)
-	ret, err := plugin.GetSourceConf(pluginName, language)
+	ret, err := meta.GetSourceConf(pluginName, language)
 	if err != nil {
 		handleError(w, err, "", logger)
 		return
@@ -776,7 +772,7 @@ func sourceConfKeysHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	pluginName := vars["name"]
-	ret := plugin.GetSourceConfKeys(pluginName)
+	ret := meta.GetSourceConfKeys(pluginName)
 	if nil != ret {
 		jsonResponse(ret, w, logger)
 		return
@@ -795,14 +791,14 @@ func sourceConfKeyHandler(w http.ResponseWriter, r *http.Request) {
 	language := getLanguage(r)
 	switch r.Method {
 	case http.MethodDelete:
-		err = plugin.DelSourceConfKey(pluginName, confKey, language)
+		err = meta.DelSourceConfKey(pluginName, confKey, language)
 	case http.MethodPost:
 		v, err := ioutil.ReadAll(r.Body)
 		if err != nil {
 			handleError(w, err, "Invalid body", logger)
 			return
 		}
-		err = plugin.AddSourceConfKey(pluginName, confKey, language, v)
+		err = meta.AddSourceConfKey(pluginName, confKey, language, v)
 	}
 	if err != nil {
 		handleError(w, err, "", logger)
@@ -831,9 +827,9 @@ func sourceConfKeyFieldsHandler(w http.ResponseWriter, r *http.Request) {
 	language := getLanguage(r)
 	switch r.Method {
 	case http.MethodDelete:
-		err = plugin.DelSourceConfKeyField(pluginName, confKey, language, v)
+		err = meta.DelSourceConfKeyField(pluginName, confKey, language, v)
 	case http.MethodPost:
-		err = plugin.AddSourceConfKeyField(pluginName, confKey, language, v)
+		err = meta.AddSourceConfKeyField(pluginName, confKey, language, v)
 	}
 	if err != nil {
 		handleError(w, err, "", logger)
@@ -854,6 +850,7 @@ func getLanguage(r *http.Request) string {
 
 func servicesHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
+	serviceManager := service.GetManager()
 	switch r.Method {
 	case http.MethodGet:
 		content, err := serviceManager.List()
@@ -884,7 +881,7 @@ func serviceHandler(w http.ResponseWriter, r *http.Request) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
-
+	serviceManager := service.GetManager()
 	switch r.Method {
 	case http.MethodDelete:
 		err := serviceManager.Delete(name)
@@ -922,6 +919,7 @@ func serviceHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func serviceFunctionsHandler(w http.ResponseWriter, r *http.Request) {
+	serviceManager := service.GetManager()
 	content, err := serviceManager.ListFunctions()
 	if err != nil {
 		handleError(w, err, "service list command error", logger)
@@ -933,6 +931,7 @@ func serviceFunctionsHandler(w http.ResponseWriter, r *http.Request) {
 func serviceFunctionHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	name := vars["name"]
+	serviceManager := service.GetManager()
 	j, err := serviceManager.GetFunction(name)
 	if err != nil {
 		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)

internal/server/rpc.go

@@ -19,7 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/lf-edge/ekuiper/internal/pkg/model"
-	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/native"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/lf-edge/ekuiper/internal/topo/sink"
 	"strings"
@@ -58,7 +58,7 @@ func stopQuery() {
 /**
  * qid is not currently used.
  */
-func (t *Server) GetQueryResult(qid string, reply *string) error {
+func (t *Server) GetQueryResult(_ string, reply *string) error {
 	if rs, ok := registry.Load(QueryRuleId); ok {
 		c := (*rs.Topology).GetContext()
 		if c != nil && c.Err() != nil {
@@ -202,7 +202,7 @@ func (t *Server) DropRule(name string, reply *string) error {
 }
 
 func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
-	pt := plugin.PluginType(arg.Type)
+	pt := native.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
@@ -210,7 +210,7 @@ func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
 	if p.GetFile() == "" {
 		return fmt.Errorf("Create plugin error: Missing plugin file url.")
 	}
-	err = pluginManager.Register(pt, p)
+	err = native.GetManager().Register(pt, p)
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
 	} else {
@@ -220,14 +220,14 @@ func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) RegisterPlugin(arg *model.PluginDesc, reply *string) error {
-	p, err := getPluginByJson(arg, plugin.FUNCTION)
+	p, err := getPluginByJson(arg, native.FUNCTION)
 	if err != nil {
 		return fmt.Errorf("Register plugin functions error: %s", err)
 	}
 	if len(p.GetSymbols()) == 0 {
 		return fmt.Errorf("Register plugin functions error: Missing function list.")
 	}
-	err = pluginManager.RegisterFuncs(p.GetName(), p.GetSymbols())
+	err = native.GetManager().RegisterFuncs(p.GetName(), p.GetSymbols())
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
 	} else {
@@ -237,12 +237,12 @@ func (t *Server) RegisterPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) DropPlugin(arg *model.PluginDesc, reply *string) error {
-	pt := plugin.PluginType(arg.Type)
+	pt := native.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Drop plugin error: %s", err)
 	}
-	err = pluginManager.Delete(pt, p.GetName(), arg.Stop)
+	err = native.GetManager().Delete(pt, p.GetName(), arg.Stop)
 	if err != nil {
 		return fmt.Errorf("Drop plugin error: %s", err)
 	} else {
@@ -257,39 +257,31 @@ func (t *Server) DropPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) ShowPlugins(arg int, reply *string) error {
-	pt := plugin.PluginType(arg)
-	l, err := pluginManager.List(pt)
-	if err != nil {
-		return fmt.Errorf("Show plugin error: %s", err)
-	} else {
-		if len(l) == 0 {
-			l = append(l, "No plugin is found.")
-		}
-		*reply = strings.Join(l, "\n")
+	pt := native.PluginType(arg)
+	l := native.GetManager().List(pt)
+	if len(l) == 0 {
+		l = append(l, "No plugin is found.")
 	}
+	*reply = strings.Join(l, "\n")
 	return nil
 }
 
 func (t *Server) ShowUdfs(_ int, reply *string) error {
-	l, err := pluginManager.ListSymbols()
-	if err != nil {
-		return fmt.Errorf("Show UDFs error: %s", err)
-	} else {
-		if len(l) == 0 {
-			l = append(l, "No udf is found.")
-		}
-		*reply = strings.Join(l, "\n")
+	l := native.GetManager().ListSymbols()
+	if len(l) == 0 {
+		l = append(l, "No udf is found.")
 	}
+	*reply = strings.Join(l, "\n")
 	return nil
 }
 
 func (t *Server) DescPlugin(arg *model.PluginDesc, reply *string) error {
-	pt := plugin.PluginType(arg.Type)
+	pt := native.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Describe plugin error: %s", err)
 	}
-	m, ok := pluginManager.Get(pt, p.GetName())
+	m, ok := native.GetManager().GetPluginInfo(pt, p.GetName())
 	if !ok {
 		return fmt.Errorf("Describe plugin error: not found")
 	} else {
@@ -303,7 +295,7 @@ func (t *Server) DescPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) DescUdf(arg string, reply *string) error {
-	m, ok := pluginManager.GetSymbol(arg)
+	m, ok := native.GetManager().GetPluginBySymbol(native.FUNCTION, arg)
 	if !ok {
 		return fmt.Errorf("Describe udf error: not found")
 	} else {
@@ -333,7 +325,7 @@ func (t *Server) CreateService(arg *model.RPCArgDesc, reply *string) error {
 	if sd.File == "" {
 		return fmt.Errorf("Create service error: Missing service file url.")
 	}
-	err := serviceManager.Create(sd)
+	err := service.GetManager().Create(sd)
 	if err != nil {
 		return fmt.Errorf("Create service error: %s", err)
 	} else {
@@ -343,7 +335,7 @@ func (t *Server) CreateService(arg *model.RPCArgDesc, reply *string) error {
 }
 
 func (t *Server) DescService(name string, reply *string) error {
-	s, err := serviceManager.Get(name)
+	s, err := service.GetManager().Get(name)
 	if err != nil {
 		return fmt.Errorf("Desc service error : %s.", err)
 	} else {
@@ -357,7 +349,7 @@ func (t *Server) DescService(name string, reply *string) error {
 }
 
 func (t *Server) DescServiceFunc(name string, reply *string) error {
-	s, err := serviceManager.GetFunction(name)
+	s, err := service.GetManager().GetFunction(name)
 	if err != nil {
 		return fmt.Errorf("Desc service func error : %s.", err)
 	} else {
@@ -371,7 +363,7 @@ func (t *Server) DescServiceFunc(name string, reply *string) error {
 }
 
 func (t *Server) DropService(name string, reply *string) error {
-	err := serviceManager.Delete(name)
+	err := service.GetManager().Delete(name)
 	if err != nil {
 		return fmt.Errorf("Drop service error : %s.", err)
 	}
@@ -380,7 +372,7 @@ func (t *Server) DropService(name string, reply *string) error {
 }
 
 func (t *Server) ShowServices(_ int, reply *string) error {
-	s, err := serviceManager.List()
+	s, err := service.GetManager().List()
 	if err != nil {
 		return fmt.Errorf("Show service error: %s.", err)
 	}
@@ -397,7 +389,7 @@ func (t *Server) ShowServices(_ int, reply *string) error {
 }
 
 func (t *Server) ShowServiceFuncs(_ int, reply *string) error {
-	s, err := serviceManager.ListFunctions()
+	s, err := service.GetManager().ListFunctions()
 	if err != nil {
 		return fmt.Errorf("Show service funcs error: %s.", err)
 	}
@@ -425,8 +417,8 @@ func marshalDesc(m interface{}) (string, error) {
 	return dst.String(), nil
 }
 
-func getPluginByJson(arg *model.PluginDesc, pt plugin.PluginType) (plugin.Plugin, error) {
-	p := plugin.NewPluginByType(pt)
+func getPluginByJson(arg *model.PluginDesc, pt native.PluginType) (native.Plugin, error) {
+	p := native.NewPluginByType(pt)
 	if arg.Json != "" {
 		if err := json.Unmarshal([]byte(arg.Json), p); err != nil {
 			return nil, fmt.Errorf("Parse plugin %s error : %s.", arg.Json, err)
@@ -441,17 +433,19 @@ func init() {
 	go func() {
 		for {
 			<-ticker.C
-			if _, ok := registry.Load(QueryRuleId); !ok {
-				continue
-			}
-
-			n := time.Now()
-			w := 10 * time.Second
-			if v := n.Sub(sink.QR.LastFetch); v >= w {
-				logger.Printf("The client seems no longer fetch the query result, stop the query now.")
-				stopQuery()
-				ticker.Stop()
-				return
+			if registry != nil {
+				if _, ok := registry.Load(QueryRuleId); !ok {
+					continue
+				}
+
+				n := time.Now()
+				w := 10 * time.Second
+				if v := n.Sub(sink.QR.LastFetch); v >= w {
+					logger.Printf("The client seems no longer fetch the query result, stop the query now.")
+					stopQuery()
+					ticker.Stop()
+					return
+				}
 			}
 		}
 	}()

internal/server/server.go

@@ -15,12 +15,15 @@
 package server
 
 import (
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
+	"github.com/lf-edge/ekuiper/internal/binder/meta"
 	"github.com/lf-edge/ekuiper/internal/conf"
 	"github.com/lf-edge/ekuiper/internal/pkg/store"
-	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/native"
 	"github.com/lf-edge/ekuiper/internal/processor"
 	"github.com/lf-edge/ekuiper/internal/service"
-	"github.com/lf-edge/ekuiper/internal/xsql"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"context"
@@ -39,8 +42,6 @@ var (
 	version         = ""
 	ruleProcessor   *processor.RuleProcessor
 	streamProcessor *processor.StreamProcessor
-	pluginManager   *plugin.Manager
-	serviceManager  *service.Manager
 )
 
 func StartUp(Version, LoadFileType string) {
@@ -56,15 +57,29 @@ func StartUp(Version, LoadFileType string) {
 
 	ruleProcessor = processor.NewRuleProcessor()
 	streamProcessor = processor.NewStreamProcessor()
-	pluginManager, err = plugin.NewPluginManager()
+
+	// Bind the source, function, sink
+	nativeManager, err := native.InitManager()
+	if err != nil {
+		panic(err)
+	}
+	serviceManager, err := service.InitManager()
+	if err != nil {
+		panic(err)
+	}
+	entries := []binder.FactoryEntry{
+		{Name: "native plugin", Factory: nativeManager},
+		{Name: "external service", Factory: serviceManager},
+	}
+	err = function.Initialize(entries)
 	if err != nil {
 		panic(err)
 	}
-	serviceManager, err = service.GetServiceManager()
+	err = io.Initialize(entries)
 	if err != nil {
 		panic(err)
 	}
-	xsql.InitFuncRegisters(serviceManager, pluginManager)
+	meta.Bind()
 
 	registry = &RuleRegistry{internal: make(map[string]*RuleState)}
 

internal/service/manager.go

@@ -48,7 +48,7 @@ type Manager struct {
 	functionKV kv.KeyValue
 }
 
-func GetServiceManager() (*Manager, error) {
+func InitManager() (*Manager, error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 	if singleton == nil {
@@ -85,6 +85,10 @@ func GetServiceManager() (*Manager, error) {
 	return singleton, nil
 }
 
+func GetManager() *Manager {
+	return singleton
+}
+
 /**
  * This function will parse the service definition json files in etc/services.
  * It will validate all json files and their schemaFiles. If invalid, it just prints
@@ -176,12 +180,10 @@ func (m *Manager) initFile(baseName string) error {
 	return nil
 }
 
-// Start Implement FunctionRegister
+// Start Implement FunctionFactory
 
-func (m *Manager) HasFunction(name string) bool {
-	_, ok := m.getFunction(name)
-	kconf.Log.Debugf("found external function %s? %v ", name, ok)
-	return ok
+func (m *Manager) HasFunctionSet(name string) bool {
+	return false
 }
 
 func (m *Manager) Function(name string) (api.Function, error) {
@@ -205,7 +207,7 @@ func (m *Manager) Function(name string) (api.Function, error) {
 	return &ExternalFunc{exe: e, methodName: f.MethodName}, nil
 }
 
-// End Implement FunctionRegister
+// End Implement FunctionFactory
 
 func (m *Manager) HasService(name string) bool {
 	_, ok := m.getService(name)

internal/service/manager_test.go

@@ -15,7 +15,8 @@
 package service
 
 import (
-	"github.com/lf-edge/ekuiper/internal/xsql"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
 	"reflect"
 	"testing"
 )
@@ -23,9 +24,16 @@ import (
 var m *Manager
 
 func init() {
-	m, _ = GetServiceManager()
+	serviceManager, err := InitManager()
+	if err != nil {
+		panic(err)
+	}
+	err = function.Initialize([]binder.FactoryEntry{{Name: "external service", Factory: serviceManager}})
+	if err != nil {
+		panic(err)
+	}
+	m = GetManager()
 	m.InitByFiles()
-	xsql.InitFuncRegisters(m)
 }
 
 func TestInitByFiles(t *testing.T) {

internal/topo/node/operations.go

@@ -36,16 +36,14 @@ func (f UnFunc) Apply(ctx api.StreamContext, data interface{}) interface{} {
 
 type UnaryOperator struct {
 	*defaultSinkNode
-	op            UnOperation
-	funcRegisters []xsql.FunctionRegister
-	mutex         sync.RWMutex
-	cancelled     bool
+	op        UnOperation
+	mutex     sync.RWMutex
+	cancelled bool
 }
 
 // NewUnary creates *UnaryOperator value
-func New(name string, registers []xsql.FunctionRegister, options *api.RuleOption) *UnaryOperator {
+func New(name string, options *api.RuleOption) *UnaryOperator {
 	return &UnaryOperator{
-		funcRegisters: registers,
 		defaultSinkNode: &defaultSinkNode{
 			input: make(chan interface{}, options.BufferLength),
 			defaultNode: &defaultNode{
@@ -108,7 +106,7 @@ func (o *UnaryOperator) doOp(ctx api.StreamContext, errCh chan<- error) {
 	o.mutex.Lock()
 	o.statManagers = append(o.statManagers, stats)
 	o.mutex.Unlock()
-	fv, afv := xsql.NewFunctionValuersForOp(exeCtx, o.funcRegisters)
+	fv, afv := xsql.NewFunctionValuersForOp(exeCtx)
 
 	for {
 		select {

internal/topo/node/sink_node.go

@@ -18,10 +18,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
 	"github.com/lf-edge/ekuiper/internal/conf"
-	"github.com/lf-edge/ekuiper/internal/plugin"
 	ct "github.com/lf-edge/ekuiper/internal/template"
-	"github.com/lf-edge/ekuiper/internal/topo/sink"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/cast"
 	"sync"
@@ -405,33 +404,25 @@ func doCollectCacheTuple(sink api.Sink, item *CacheTuple, stats StatManager, ret
 	}
 }
 
-func doGetSink(name string, action map[string]interface{}) (api.Sink, error) {
+func getSink(name string, action map[string]interface{}) (api.Sink, error) {
 	var (
 		s   api.Sink
 		err error
 	)
-	switch name {
-	case "log":
-		s = sink.NewLogSink()
-	case "logToMemory":
-		s = sink.NewLogSinkToMemory()
-	case "mqtt":
-		s = &sink.MQTTSink{}
-	case "rest":
-		s = &sink.RestSink{}
-	case "nop":
-		s = &sink.NopSink{}
-	default:
-		s, err = plugin.GetSink(name)
+	s, err = io.Sink(name)
+	if s != nil {
+		err = s.Configure(action)
 		if err != nil {
 			return nil, err
 		}
+		return s, nil
+	} else {
+		if err != nil {
+			return nil, err
+		} else {
+			return nil, fmt.Errorf("sink %s not found", name)
+		}
 	}
-	err = s.Configure(action)
-	if err != nil {
-		return nil, err
-	}
-	return s, nil
 }
 
 //Override defaultNode

internal/topo/node/source_node.go

@@ -16,8 +16,6 @@ package node
 
 import (
 	"github.com/lf-edge/ekuiper/internal/conf"
-	"github.com/lf-edge/ekuiper/internal/plugin"
-	"github.com/lf-edge/ekuiper/internal/topo/source"
 	"github.com/lf-edge/ekuiper/internal/xsql"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/ast"
@@ -162,27 +160,6 @@ func (m *SourceNode) reset() {
 	m.statManagers = nil
 }
 
-func doGetSource(t string) (api.Source, error) {
-	var (
-		s   api.Source
-		err error
-	)
-	switch t {
-	case "mqtt":
-		s = &source.MQTTSource{}
-	case "httppull":
-		s = &source.HTTPPullSource{}
-	case "file":
-		s = &source.FileSource{}
-	default:
-		s, err = plugin.GetSource(t)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return s, nil
-}
-
 func (m *SourceNode) drainError(errCh chan<- error, err error, ctx api.StreamContext, logger api.Logger) {
 	select {
 	case errCh <- err:

internal/topo/node/source_pool.go

@@ -17,6 +17,7 @@ package node
 import (
 	"context"
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
 	"github.com/lf-edge/ekuiper/internal/conf"
 	kctx "github.com/lf-edge/ekuiper/internal/topo/context"
 	"github.com/lf-edge/ekuiper/pkg/api"
@@ -38,13 +39,18 @@ func getSourceInstance(node *SourceNode, index int) (*sourceInstance, error) {
 		rkey := fmt.Sprintf("%s.%s", node.sourceType, node.name)
 		s, ok := pool.load(rkey)
 		if !ok {
-			ns, err := getSource(node.sourceType)
-			if err != nil {
-				return nil, err
-			}
-			s, err = pool.addInstance(rkey, node, ns, index)
-			if err != nil {
-				return nil, err
+			ns, err := io.Source(node.sourceType)
+			if ns != nil {
+				s, err = pool.addInstance(rkey, node, ns, index)
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				if err != nil {
+					return nil, err
+				} else {
+					return nil, fmt.Errorf("source %s not found", node.sourceType)
+				}
 			}
 		}
 		// attach
@@ -59,15 +65,19 @@ func getSourceInstance(node *SourceNode, index int) (*sourceInstance, error) {
 			sourceInstanceChannels: s.outputs[instanceKey],
 		}
 	} else {
-		ns, err := getSource(node.sourceType)
-		if err != nil {
-			return nil, err
-		}
-		si, err = start(nil, node, ns, index)
-		if err != nil {
-			return nil, err
+		ns, err := io.Source(node.sourceType)
+		if ns != nil {
+			si, err = start(nil, node, ns, index)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			if err != nil {
+				return nil, err
+			} else {
+				return nil, fmt.Errorf("source %s not found", node.sourceType)
+			}
 		}
-
 	}
 	return si, nil
 }

internal/topo/node/sources_for_test_with_edgex.go

@@ -1,46 +0,0 @@
-// Copyright 2021 EMQ Technologies Co., Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build test
-// +build edgex
-
-package node
-
-import (
-	"github.com/lf-edge/ekuiper/internal/topo/sink"
-	"github.com/lf-edge/ekuiper/internal/topo/source"
-	"github.com/lf-edge/ekuiper/internal/topo/topotest/mocknode"
-	"github.com/lf-edge/ekuiper/pkg/api"
-)
-
-func getSource(t string) (api.Source, error) {
-	if t == "edgex" {
-		return &source.EdgexSource{}, nil
-	} else if t == "mock" {
-		return &mocknode.MockSource{}, nil
-	}
-	return doGetSource(t)
-}
-
-func getSink(name string, action map[string]interface{}) (api.Sink, error) {
-	if name == "edgex" {
-		s := &sink.EdgexMsgBusSink{}
-		if err := s.Configure(action); err != nil {
-			return nil, err
-		} else {
-			return s, nil
-		}
-	}
-	return doGetSink(name, action)
-}

internal/topo/operator/aggregate_test.go

@@ -467,7 +467,7 @@ func TestAggregatePlan_Apply(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &AggregateOp{Dimensions: stmt.Dimensions.GetGroups()}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		gr, ok := result.(xsql.GroupedTuplesSet)
@@ -538,7 +538,7 @@ func TestAggregatePlanError(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &AggregateOp{Dimensions: stmt.Dimensions.GetGroups()}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {

internal/topo/operator/filter_test.go

@@ -402,7 +402,7 @@ func TestFilterPlan_Apply(t *testing.T) {
 			t.Errorf("statement %d parse error %s", i, err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &FilterOp{Condition: stmt.Condition}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {
@@ -541,7 +541,7 @@ func TestFilterPlanError(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &FilterOp{Condition: stmt.Condition}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {

internal/topo/operator/having_test.go

@@ -339,7 +339,7 @@ func TestHavingPlan_Apply(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &HavingOp{Condition: stmt.Having}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {
@@ -510,7 +510,7 @@ func TestHavingPlanAlias_Apply(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &HavingOp{Condition: stmt.Having}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {
@@ -588,7 +588,7 @@ func TestHavingPlanError(t *testing.T) {
 			t.Errorf("statement parse error %s", err)
 			break
 		}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		pp := &HavingOp{Condition: stmt.Having}
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {

internal/topo/operator/join_multi_test.go

@@ -1448,7 +1448,7 @@ func TestMultiJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {

internal/topo/operator/join_test.go

@@ -940,7 +940,7 @@ func TestLeftJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
@@ -1564,7 +1564,7 @@ func TestInnerJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
@@ -1842,7 +1842,7 @@ func TestRightJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
@@ -2187,7 +2187,7 @@ func TestFullJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
@@ -2410,7 +2410,7 @@ func TestCrossJoinPlan_Apply(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
@@ -2486,7 +2486,7 @@ func TestCrossJoinPlanError(t *testing.T) {
 		if table, ok := stmt.Sources[0].(*ast.Table); !ok {
 			t.Errorf("statement source is not a table")
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			pp := &JoinOp{Joins: stmt.Joins, From: table}
 			result := pp.Apply(ctx, tt.data, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {

internal/topo/operator/math_func_test.go

@@ -480,7 +480,7 @@ func TestMathAndConversionFunc_Apply1(t *testing.T) {
 			continue
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {

internal/topo/operator/misc_func_test.go

@@ -253,7 +253,7 @@ func TestMiscFunc_Apply1(t *testing.T) {
 			t.Errorf("parse sql %s error %v", tt.sql, err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -305,7 +305,7 @@ func TestMqttFunc_Apply2(t *testing.T) {
 			t.Errorf("parse sql %s error %v", tt.sql, err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -421,7 +421,7 @@ func TestMetaFunc_Apply1(t *testing.T) {
 			t.Errorf("parse sql %s error %v", tt.sql, err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -833,7 +833,7 @@ func TestJsonPathFunc_Apply1(t *testing.T) {
 			t.Errorf("parse sql %s error %v", tt.sql, err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		switch rt := result.(type) {
 		case []byte:

internal/topo/operator/order_test.go

@@ -578,7 +578,7 @@ func TestOrderPlan_Apply(t *testing.T) {
 		}
 
 		pp := &OrderOp{SortFields: stmt.SortFields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {
 			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)

internal/topo/operator/preprocessor_test.go

@@ -581,7 +581,7 @@ func TestPreprocessor_Apply(t *testing.T) {
 			return
 		} else {
 			tuple := &xsql.Tuple{Message: dm}
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			result := pp.Apply(ctx, tuple, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
 				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tuple, tt.result, result)
@@ -717,7 +717,7 @@ func TestPreprocessorTime_Apply(t *testing.T) {
 			return
 		} else {
 			tuple := &xsql.Tuple{Message: dm}
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			result := pp.Apply(ctx, tuple, fv, afv)
 			//workaround make sure all the timezone are the same for time vars or the DeepEqual will be false.
 			if rt, ok := result.(*xsql.Tuple); ok {
@@ -907,7 +907,7 @@ func TestPreprocessorEventtime_Apply(t *testing.T) {
 			return
 		} else {
 			tuple := &xsql.Tuple{Message: dm}
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			result := pp.Apply(ctx, tuple, fv, afv)
 			//workaround make sure all the timezone are the same for time vars or the DeepEqual will be false.
 			if rt, ok := result.(*xsql.Tuple); ok {
@@ -987,7 +987,7 @@ func TestPreprocessorError(t *testing.T) {
 			return
 		} else {
 			tuple := &xsql.Tuple{Message: dm}
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			result := pp.Apply(ctx, tuple, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
 				t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tuple, tt.result, result)
@@ -1118,7 +1118,7 @@ func TestPreprocessorForBinary(t *testing.T) {
 			return
 		} else {
 			tuple := &xsql.Tuple{Message: dm}
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			result := pp.Apply(ctx, tuple, fv, afv)
 			if !reflect.DeepEqual(tt.result, result) {
 				t.Errorf("%d. %q\n\nresult mismatch", i, tuple)

internal/topo/operator/project_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/lf-edge/ekuiper/internal/conf"
 	"github.com/lf-edge/ekuiper/internal/topo/context"
 	"github.com/lf-edge/ekuiper/internal/xsql"
-	"github.com/lf-edge/ekuiper/pkg/ast"
 	"github.com/lf-edge/ekuiper/pkg/cast"
 	"reflect"
 	"strings"
@@ -558,7 +557,7 @@ func TestProjectPlan_Apply1(t *testing.T) {
 			continue
 		}
 		pp := &ProjectOp{Fields: stmt.Fields, SendMeta: true}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -1176,7 +1175,7 @@ func TestProjectPlan_MultiInput(t *testing.T) {
 		stmt, _ := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
 
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -1398,8 +1397,8 @@ func TestProjectPlan_Funcs(t *testing.T) {
 		if err != nil {
 			t.Error(err)
 		}
-		pp := &ProjectOp{Fields: stmt.Fields, IsAggregate: ast.IsAggStatement(stmt)}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		pp := &ProjectOp{Fields: stmt.Fields, IsAggregate: xsql.IsAggStatement(stmt)}
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -2248,7 +2247,7 @@ func TestProjectPlan_AggFuncs(t *testing.T) {
 			t.Error(err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields, IsAggregate: true}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {
@@ -2417,8 +2416,8 @@ func TestProjectPlanError(t *testing.T) {
 	for i, tt := range tests {
 		stmt, _ := xsql.NewParser(strings.NewReader(tt.sql)).Parse()
 
-		pp := &ProjectOp{Fields: stmt.Fields, IsAggregate: ast.IsAggStatement(stmt)}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		pp := &ProjectOp{Fields: stmt.Fields, IsAggregate: xsql.IsAggStatement(stmt)}
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		if !reflect.DeepEqual(tt.result, result) {
 			t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.sql, tt.result, result)

internal/topo/operator/str_func_test.go

@@ -660,7 +660,7 @@ func TestStrFunc_Apply1(t *testing.T) {
 			t.Errorf("parse sql %s error %v", tt.sql, err)
 		}
 		pp := &ProjectOp{Fields: stmt.Fields}
-		fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+		fv, afv := xsql.NewFunctionValuersForOp(nil)
 		result := pp.Apply(ctx, tt.data, fv, afv)
 		var mapRes []map[string]interface{}
 		if v, ok := result.([]byte); ok {

internal/topo/operator/table_processor_test.go

@@ -132,7 +132,7 @@ func TestTableProcessor_Apply(t *testing.T) {
 			t.Log(e)
 			t.Fail()
 		} else {
-			fv, afv := xsql.NewFunctionValuersForOp(nil, xsql.FuncRegisters)
+			fv, afv := xsql.NewFunctionValuersForOp(nil)
 			for _, m := range dm {
 				pp.Apply(ctx, &xsql.Tuple{
 					Emitter: "demo",

internal/topo/planner/analyzer.go

@@ -16,6 +16,7 @@ package planner
 
 import (
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
 	"github.com/lf-edge/ekuiper/internal/xsql"
 	"github.com/lf-edge/ekuiper/pkg/ast"
 	"github.com/lf-edge/ekuiper/pkg/kv"
@@ -110,14 +111,14 @@ func decorateStmt(s *ast.SelectStatement, store kv.KeyValue) ([]*ast.StreamStmt,
 }
 
 func validate(s *ast.SelectStatement) (err error) {
-	if ast.IsAggregate(s.Condition) {
+	if xsql.IsAggregate(s.Condition) {
 		return fmt.Errorf("Not allowed to call aggregate functions in WHERE clause.")
 	}
 	if !allAggregate(s.Having) {
 		return fmt.Errorf("Not allowed to call non-aggregate functions in HAVING clause.")
 	}
 	for _, d := range s.Dimensions {
-		if ast.IsAggregate(d.Expr) {
+		if xsql.IsAggregate(d.Expr) {
 			return fmt.Errorf("Not allowed to call aggregate functions in GROUP BY clause.")
 		}
 	}
@@ -125,9 +126,9 @@ func validate(s *ast.SelectStatement) (err error) {
 		switch f := n.(type) {
 		case *ast.Call:
 			// aggregate call should not have any aggregate arg
-			if ast.FuncFinderSingleton().IsAggFunc(f) {
+			if function.IsAggFunc(f.Name) {
 				for _, arg := range f.Args {
-					tr := ast.IsAggregate(arg)
+					tr := xsql.IsAggregate(arg)
 					if tr {
 						err = fmt.Errorf("invalid argument for func %s: aggregate argument is not allowed", f.Name)
 						return false
@@ -155,7 +156,7 @@ func allAggregate(expr ast.Expr) (r bool) {
 				return false
 			}
 		case *ast.Call, *ast.FieldRef:
-			if !ast.IsAggregate(f) {
+			if !xsql.IsAggregate(f) {
 				r = false
 				return false
 			}

internal/topo/planner/analyzer_test.go

@@ -27,6 +27,9 @@ import (
 	"testing"
 )
 
+func init() {
+}
+
 type errorStruct struct {
 	err  string
 	serr *string

internal/topo/planner/planner.go

@@ -314,7 +314,7 @@ func createLogicalPlan(stmt *ast.SelectStatement, opt *api.RuleOption, store kv.
 	if stmt.Fields != nil {
 		p = ProjectPlan{
 			fields:      stmt.Fields,
-			isAggregate: ast.IsAggStatement(stmt),
+			isAggregate: xsql.IsAggStatement(stmt),
 			sendMeta:    opt.SendMetaToSink,
 		}.Init()
 		p.SetChildren(children)
@@ -324,7 +324,7 @@ func createLogicalPlan(stmt *ast.SelectStatement, opt *api.RuleOption, store kv.
 }
 
 func Transform(op node.UnOperation, name string, options *api.RuleOption) *node.UnaryOperator {
-	unaryOperator := node.New(name, xsql.FuncRegisters, options)
+	unaryOperator := node.New(name, options)
 	unaryOperator.SetOperation(op)
 	return unaryOperator
 }

internal/topo/sink/log_sink.go

@@ -24,7 +24,7 @@ import (
 
 // NewLogSink log action, no properties now
 // example: {"log":{}}
-func NewLogSink() *collector.FuncCollector {
+func NewLogSink() api.Sink {
 	return collector.Func(func(ctx api.StreamContext, data interface{}) error {
 		log := ctx.GetLogger()
 		log.Infof("sink result for rule %s: %s", ctx.GetRuleId(), data)
@@ -40,7 +40,7 @@ type QueryResult struct {
 
 var QR = &QueryResult{LastFetch: time.Now()}
 
-func NewLogSinkToMemory() *collector.FuncCollector {
+func NewLogSinkToMemory() api.Sink {
 	QR.Results = make([]string, 10)
 	return collector.Func(func(ctx api.StreamContext, data interface{}) error {
 		QR.Mux.Lock()

internal/topo/topotest/mock_topo.go

@@ -331,9 +331,9 @@ func HandleStream(createOrDrop bool, names []string, t *testing.T) {
 				sql = `CREATE STREAM lsessionDemo (
 				) WITH (DATASOURCE="lsessionDemo", TYPE="mock", FORMAT="json");`
 			case "ext":
-				sql = "CREATE STREAM ext (count bigint) WITH (DATASOURCE=\"ext\", TYPE=\"mock\", FORMAT=\"JSON\", TYPE=\"random\", CONF_KEY=\"ext\")"
+				sql = "CREATE STREAM ext (count bigint) WITH (DATASOURCE=\"ext\", FORMAT=\"JSON\", TYPE=\"random\", CONF_KEY=\"ext\")"
 			case "ext2":
-				sql = "CREATE STREAM ext2 (count bigint) WITH (DATASOURCE=\"ext2\", TYPE=\"mock\", FORMAT=\"JSON\", TYPE=\"random\", CONF_KEY=\"dedup\")"
+				sql = "CREATE STREAM ext2 (count bigint) WITH (DATASOURCE=\"ext2\", FORMAT=\"JSON\", TYPE=\"random\", CONF_KEY=\"dedup\")"
 			case "text":
 				sql = "CREATE STREAM text (slogan string, brand string) WITH (DATASOURCE=\"text\", TYPE=\"mock\", FORMAT=\"JSON\")"
 			case "binDemo":

internal/topo/topotest/plugin_rule_test.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !windows
 // +build !windows
 
 package topotest
@@ -20,30 +21,37 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
 	"github.com/lf-edge/ekuiper/internal/conf"
-	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/native"
 	"github.com/lf-edge/ekuiper/internal/topo/planner"
 	"github.com/lf-edge/ekuiper/internal/topo/topotest/mockclock"
-	"github.com/lf-edge/ekuiper/internal/xsql"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"os"
 	"testing"
 	"time"
 )
 
-var manager *plugin.Manager
-
 func init() {
-	var err error
-	manager, err = plugin.NewPluginManager()
+	nativeManager, err := native.InitManager()
+	if err != nil {
+		panic(err)
+	}
+	nativeEntry := binder.FactoryEntry{Name: "native plugin", Factory: nativeManager}
+	err = function.Initialize([]binder.FactoryEntry{nativeEntry})
+	if err != nil {
+		panic(err)
+	}
+	err = io.Initialize([]binder.FactoryEntry{nativeEntry})
 	if err != nil {
 		panic(err)
 	}
-	xsql.InitFuncRegisters(manager)
 }
 
 //This cannot be run in Windows. And the plugins must be built to so before running this
-//For Windows, run it in wsl with go test xsql/processors/extension_test.go xsql/processors/xsql_processor.go
+//For Windows, run it in wsl with go test -tags test internal/topo/topotest/plugin_rule_test.go internal/topo/topotest/mock_topo.go
 var CACHE_FILE = "cache"
 
 //Test for source, sink, func and agg func extensions

pkg/ast/checkAgg.go

@@ -12,19 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package ast
+package xsql
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/pkg/ast"
+)
 
 // IsAggregate check if an expression is aggregate with the binding alias info
-func IsAggregate(expr Expr) (r bool) {
-	WalkFunc(expr, func(n Node) bool {
+func IsAggregate(expr ast.Expr) (r bool) {
+	ast.WalkFunc(expr, func(n ast.Node) bool {
 		switch f := n.(type) {
-		case *Call:
-			if ok := FuncFinderSingleton().IsAggFunc(f); ok {
+		case *ast.Call:
+			if ok := function.IsAggFunc(f.Name); ok {
 				r = true
 				return false
 			}
-		case *FieldRef:
-			if f.IsAggregate() {
+		case *ast.FieldRef:
+			// lazy calculate
+			if getOrCalculateAgg(f) {
 				r = true
 				return false
 			}
@@ -34,7 +40,20 @@ func IsAggregate(expr Expr) (r bool) {
 	return
 }
 
-func IsAggStatement(stmt *SelectStatement) bool {
+func getOrCalculateAgg(f *ast.FieldRef) bool {
+	if f.IsAlias() {
+		p := f.IsAggregate
+		if p == nil {
+			tr := IsAggregate(f.Expression)
+			p = &tr
+			f.IsAggregate = p
+		}
+		return *p
+	}
+	return false
+}
+
+func IsAggStatement(stmt *ast.SelectStatement) bool {
 	if stmt.Dimensions != nil {
 		ds := stmt.Dimensions.GetGroups()
 		if ds != nil && len(ds) > 0 {
@@ -42,10 +61,10 @@ func IsAggStatement(stmt *SelectStatement) bool {
 		}
 	}
 	r := false
-	WalkFunc(stmt.Fields, func(n Node) bool {
+	ast.WalkFunc(stmt.Fields, func(n ast.Node) bool {
 		switch f := n.(type) {
-		case *Call:
-			if ok := FuncFinderSingleton().IsAggFunc(f); ok {
+		case *ast.Call:
+			if ok := function.IsAggFunc(f.Name); ok {
 				r = true
 				return false
 			}
@@ -55,14 +74,14 @@ func IsAggStatement(stmt *SelectStatement) bool {
 	return r
 }
 
-func HasAggFuncs(node Node) bool {
+func HasAggFuncs(node ast.Node) bool {
 	if node == nil {
 		return false
 	}
 	var r = false
-	WalkFunc(node, func(n Node) bool {
-		if f, ok := n.(*Call); ok {
-			if ok := FuncFinderSingleton().IsAggFunc(f); ok {
+	ast.WalkFunc(node, func(n ast.Node) bool {
+		if f, ok := n.(*ast.Call); ok {
+			if ok := function.IsAggFunc(f.Name); ok {
 				r = true
 				return false
 			}

internal/xsql/funcValuer.go

@@ -15,18 +15,10 @@
 package xsql
 
 import (
-	"github.com/lf-edge/ekuiper/pkg/api"
-	"github.com/lf-edge/ekuiper/pkg/ast"
 	"github.com/lf-edge/ekuiper/pkg/errorx"
-	"strings"
 )
 
-type FunctionRegister interface {
-	HasFunction(name string) bool
-	Function(name string) (api.Function, error)
-}
-
-// ONLY use NewFunctionValuer function to initialize
+// FunctionValuer ONLY use NewFunctionValuer function to initialize
 type FunctionValuer struct {
 	runtime *funcRuntime
 }
@@ -52,39 +44,14 @@ func (*FunctionValuer) AppendAlias(string, interface{}) bool {
 }
 
 func (fv *FunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
-	lowerName := strings.ToLower(name)
-	switch ast.FuncFinderSingleton().FuncType(lowerName) {
-	case ast.NotFoundFunc:
-		nf, fctx, err := fv.runtime.Get(name)
-		switch err {
-		case errorx.NotFoundErr:
-			return nil, false
-		case nil:
-			// do nothing, continue
-		default:
-			return err, false
-		}
-		if nf.IsAggregate() {
-			return nil, false
-		}
-		logger := fctx.GetLogger()
-		logger.Debugf("run func %s", name)
-		return nf.Exec(args, fctx)
-	case ast.AggFunc:
+	nf, fctx, err := fv.runtime.Get(name)
+	switch err {
+	case errorx.NotFoundErr:
 		return nil, false
-	case ast.MathFunc:
-		return mathCall(lowerName, args)
-	case ast.ConvFunc:
-		return convCall(lowerName, args)
-	case ast.StrFunc:
-		return strCall(lowerName, args)
-	case ast.HashFunc:
-		return hashCall(lowerName, args)
-	case ast.JsonFunc:
-		return jsonCall(lowerName, args)
-	case ast.OtherFunc:
-		return otherCall(lowerName, args)
+	case nil:
+		// do nothing, continue
 	default:
-		return nil, false
+		return err, false
 	}
+	return ExecFunc(name, nf, args, fctx)
 }

internal/xsql/func_invoker.go

@@ -0,0 +1,61 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xsql
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"github.com/lf-edge/ekuiper/pkg/ast"
+)
+
+func validateFuncs(funcName string, args []ast.Expr) error {
+	f, err := function.Function(funcName)
+	if f != nil {
+		var targs []interface{}
+		for _, arg := range args {
+			targs = append(targs, arg)
+		}
+		if mf, ok := f.(MultiFunc); ok {
+			return mf.ValidateWithName(args, funcName)
+		} else {
+			return f.Validate(targs)
+		}
+	} else {
+		if err != nil {
+			return err
+		} else {
+			return fmt.Errorf("function %s not found", funcName)
+		}
+	}
+}
+
+func ExecFunc(funcName string, f api.Function, args []interface{}, fctx api.FunctionContext) (interface{}, bool) {
+	var targs []interface{}
+	for _, arg := range args {
+		targs = append(targs, arg)
+	}
+	if mf, ok := f.(MultiFunc); ok {
+		return mf.ExecWithName(args, fctx, funcName)
+	} else {
+		return f.Exec(args, fctx)
+	}
+}
+
+// MultiFunc hack for builtin functions that works for multiple functions
+type MultiFunc interface {
+	ValidateWithName(args []ast.Expr, name string) error
+	ExecWithName(args []interface{}, ctx api.FunctionContext, name string) (interface{}, bool)
+}

internal/xsql/funcsAggregate.go

@@ -15,10 +15,8 @@
 package xsql
 
 import (
-	"fmt"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/errorx"
-	"strings"
 )
 
 type AggregateFunctionValuer struct {
@@ -26,8 +24,8 @@ type AggregateFunctionValuer struct {
 	fv   *FunctionValuer
 }
 
-func NewFunctionValuersForOp(ctx api.StreamContext, registers []FunctionRegister) (*FunctionValuer, *AggregateFunctionValuer) {
-	p := NewFuncRuntime(ctx, registers)
+func NewFunctionValuersForOp(ctx api.StreamContext) (*FunctionValuer, *AggregateFunctionValuer) {
+	p := NewFuncRuntime(ctx)
 	return NewAggregateFunctionValuers(p)
 }
 
@@ -67,311 +65,18 @@ func (*AggregateFunctionValuer) AppendAlias(string, interface{}) bool {
 }
 
 func (v *AggregateFunctionValuer) Call(name string, args []interface{}) (interface{}, bool) {
-	lowerName := strings.ToLower(name)
-	switch lowerName {
-	case "avg":
-		arg0 := args[0].([]interface{})
-		c := getCount(arg0)
-		if c > 0 {
-			v := getFirstValidArg(arg0)
-			switch v.(type) {
-			case int, int64:
-				if r, err := sliceIntTotal(arg0); err != nil {
-					return err, false
-				} else {
-					return r / c, true
-				}
-			case float64:
-				if r, err := sliceFloatTotal(arg0); err != nil {
-					return err, false
-				} else {
-					return r / float64(c), true
-				}
-			case nil:
-				return nil, true
-			default:
-				return fmt.Errorf("run avg function error: found invalid arg %[1]T(%[1]v)", v), false
-			}
-		}
-		return 0, true
-	case "count":
-		arg0 := args[0].([]interface{})
-		return getCount(arg0), true
-	case "max":
-		arg0 := args[0].([]interface{})
-		if len(arg0) > 0 {
-			v := getFirstValidArg(arg0)
-			switch t := v.(type) {
-			case int:
-				if r, err := sliceIntMax(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case int64:
-				if r, err := sliceIntMax(arg0, int(t)); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case float64:
-				if r, err := sliceFloatMax(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case string:
-				if r, err := sliceStringMax(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case nil:
-				return nil, true
-			default:
-				return fmt.Errorf("run max function error: found invalid arg %[1]T(%[1]v)", v), false
-			}
-		}
-		return fmt.Errorf("run max function error: empty data"), false
-	case "min":
-		arg0 := args[0].([]interface{})
-		if len(arg0) > 0 {
-			v := getFirstValidArg(arg0)
-			switch t := v.(type) {
-			case int:
-				if r, err := sliceIntMin(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case int64:
-				if r, err := sliceIntMin(arg0, int(t)); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case float64:
-				if r, err := sliceFloatMin(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case string:
-				if r, err := sliceStringMin(arg0, t); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case nil:
-				return nil, true
-			default:
-				return fmt.Errorf("run min function error: found invalid arg %[1]T(%[1]v)", v), false
-			}
-		}
-		return fmt.Errorf("run min function error: empty data"), false
-	case "sum":
-		arg0 := args[0].([]interface{})
-		if len(arg0) > 0 {
-			v := getFirstValidArg(arg0)
-			switch v.(type) {
-			case int, int64:
-				if r, err := sliceIntTotal(arg0); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case float64:
-				if r, err := sliceFloatTotal(arg0); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			case nil:
-				return nil, true
-			default:
-				return fmt.Errorf("run sum function error: found invalid arg %[1]T(%[1]v)", v), false
-			}
-		}
-		return 0, true
-	case "collect":
-		return args[0], true
-	case "deduplicate":
-		v1, ok1 := args[0].([]interface{})
-		v2, ok2 := args[1].([]interface{})
-		v3a, ok3 := args[2].([]interface{})
-
-		if ok1 && ok2 && ok3 && len(v3a) > 0 {
-			v3, ok4 := getFirstValidArg(v3a).(bool)
-			if ok4 {
-				if r, err := dedup(v1, v2, v3); err != nil {
-					return err, false
-				} else {
-					return r, true
-				}
-			}
-		}
-		return fmt.Errorf("Invalid argument type found."), false
+	nf, fctx, err := v.fv.runtime.Get(name)
+	switch err {
+	case errorx.NotFoundErr:
+		return nil, false
+	case nil:
+		// do nothing, continue
 	default:
-		nf, fctx, err := v.fv.runtime.Get(name)
-		switch err {
-		case errorx.NotFoundErr:
-			return nil, false
-		case nil:
-			// do nothing, continue
-		default:
-			return err, false
-		}
-		if !nf.IsAggregate() {
-			return nil, false
-		}
-		logger := fctx.GetLogger()
-		logger.Debugf("run aggregate func %s", name)
-		return nf.Exec(args, fctx)
-	}
-}
-
-func getCount(s []interface{}) int {
-	c := 0
-	for _, v := range s {
-		if v != nil {
-			c++
-		}
+		return err, false
 	}
-	return c
+	return ExecFunc(name, nf, args, fctx)
 }
 
 func (v *AggregateFunctionValuer) GetAllTuples() AggregateData {
 	return v.data
 }
-
-func getFirstValidArg(s []interface{}) interface{} {
-	for _, v := range s {
-		if v != nil {
-			return v
-		}
-	}
-	return nil
-}
-
-func sliceIntTotal(s []interface{}) (int, error) {
-	var total int
-	for _, v := range s {
-		if vi, ok := v.(int); ok {
-			total += vi
-		} else if v != nil {
-			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
-		}
-	}
-	return total, nil
-}
-
-func sliceFloatTotal(s []interface{}) (float64, error) {
-	var total float64
-	for _, v := range s {
-		if vf, ok := v.(float64); ok {
-			total += vf
-		} else if v != nil {
-			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
-		}
-	}
-	return total, nil
-}
-func sliceIntMax(s []interface{}, max int) (int, error) {
-	for _, v := range s {
-		if vi, ok := v.(int); ok {
-			if max < vi {
-				max = vi
-			}
-		} else if v != nil {
-			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
-		}
-	}
-	return max, nil
-}
-func sliceFloatMax(s []interface{}, max float64) (float64, error) {
-	for _, v := range s {
-		if vf, ok := v.(float64); ok {
-			if max < vf {
-				max = vf
-			}
-		} else if v != nil {
-			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
-		}
-	}
-	return max, nil
-}
-
-func sliceStringMax(s []interface{}, max string) (string, error) {
-	for _, v := range s {
-		if vs, ok := v.(string); ok {
-			if max < vs {
-				max = vs
-			}
-		} else if v != nil {
-			return "", fmt.Errorf("requires string but found %[1]T(%[1]v)", v)
-		}
-	}
-	return max, nil
-}
-func sliceIntMin(s []interface{}, min int) (int, error) {
-	for _, v := range s {
-		if vi, ok := v.(int); ok {
-			if min > vi {
-				min = vi
-			}
-		} else if v != nil {
-			return 0, fmt.Errorf("requires int but found %[1]T(%[1]v)", v)
-		}
-	}
-	return min, nil
-}
-func sliceFloatMin(s []interface{}, min float64) (float64, error) {
-	for _, v := range s {
-		if vf, ok := v.(float64); ok {
-			if min > vf {
-				min = vf
-			}
-		} else if v != nil {
-			return 0, fmt.Errorf("requires float64 but found %[1]T(%[1]v)", v)
-		}
-	}
-	return min, nil
-}
-
-func sliceStringMin(s []interface{}, min string) (string, error) {
-	for _, v := range s {
-		if vs, ok := v.(string); ok {
-			if min < vs {
-				min = vs
-			}
-		} else if v != nil {
-			return "", fmt.Errorf("requires string but found %[1]T(%[1]v)", v)
-		}
-	}
-	return min, nil
-}
-
-func dedup(r []interface{}, col []interface{}, all bool) (interface{}, error) {
-	keyset := make(map[string]bool)
-	result := make([]interface{}, 0)
-	for i, m := range col {
-		key := fmt.Sprintf("%v", m)
-		if _, ok := keyset[key]; !ok {
-			if all {
-				result = append(result, r[i])
-			} else if i == len(col)-1 {
-				result = append(result, r[i])
-			}
-			keyset[key] = true
-		}
-	}
-	if !all {
-		if len(result) == 0 {
-			return nil, nil
-		} else {
-			return result[0], nil
-		}
-	} else {
-		return result, nil
-	}
-}

internal/xsql/functionRuntime.go

@@ -15,6 +15,7 @@
 package xsql
 
 import (
+	"github.com/lf-edge/ekuiper/internal/binder/function"
 	"github.com/lf-edge/ekuiper/internal/topo/context"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/errorx"
@@ -25,9 +26,8 @@ import (
 //Each operator has a single instance of this to hold the context
 type funcRuntime struct {
 	sync.Mutex
-	regs          map[string]*funcReg
-	parentCtx     api.StreamContext
-	funcRegisters []FunctionRegister
+	regs      map[string]*funcReg
+	parentCtx api.StreamContext
 }
 
 type funcReg struct {
@@ -35,10 +35,9 @@ type funcReg struct {
 	ctx api.FunctionContext
 }
 
-func NewFuncRuntime(ctx api.StreamContext, registers []FunctionRegister) *funcRuntime {
+func NewFuncRuntime(ctx api.StreamContext) *funcRuntime {
 	return &funcRuntime{
-		parentCtx:     ctx,
-		funcRegisters: registers,
+		parentCtx: ctx,
 	}
 }
 
@@ -54,17 +53,13 @@ func (fp *funcRuntime) Get(name string) (api.Function, api.FunctionContext, erro
 			err error
 		)
 		// Check service extension and plugin extension if set
-		for _, r := range fp.funcRegisters {
-			if r.HasFunction(name) {
-				nf, err = r.Function(name)
-				if err != nil {
-					return nil, nil, err
-				}
-				break
-			}
-		}
+		nf, err = function.Function(name)
 		if nf == nil {
-			return nil, nil, errorx.NotFoundErr
+			if err == nil {
+				return nil, nil, errorx.NotFoundErr
+			} else {
+				return nil, nil, err
+			}
 		}
 		fctx := context.NewDefaultFuncContext(fp.parentCtx, len(fp.regs))
 		fp.regs[name] = &funcReg{

internal/xsql/manager.go

@@ -21,7 +21,6 @@ import (
 
 var (
 	Language          = &ParseTree{}
-	FuncRegisters     []FunctionRegister
 	parserFuncRuntime *funcRuntime
 )
 
@@ -81,12 +80,4 @@ func init() {
 	Language.Handle(ast.DROP, func(p *Parser) (statement ast.Statement, e error) {
 		return p.parseDropStmt()
 	})
-
-	InitFuncRegisters()
-}
-
-func InitFuncRegisters(registers ...FunctionRegister) {
-	FuncRegisters = registers
-	parserFuncRuntime = NewFuncRuntime(nil, registers)
-	ast.InitFuncFinder(parserFuncRuntime)
 }

internal/xsql/parser_agg_test.go

@@ -17,7 +17,6 @@ package xsql
 import (
 	"fmt"
 	"github.com/lf-edge/ekuiper/internal/testx"
-	"github.com/lf-edge/ekuiper/pkg/ast"
 	"reflect"
 	"strings"
 	"testing"
@@ -48,7 +47,7 @@ func TestIsAggStatement(t *testing.T) {
 	for i, tt := range tests {
 		//fmt.Printf("Parsing SQL %q.\n", tt.s)
 		stmt, err := NewParser(strings.NewReader(tt.s)).Parse()
-		isAgg := ast.IsAggStatement(stmt)
+		isAgg := IsAggStatement(stmt)
 		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
 			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.s, tt.err, err)
 		} else if tt.err == "" && (tt.agg != isAgg) {

internal/xsql/parser_test.go

@@ -1241,7 +1241,7 @@ func TestParser_ParseStatement(t *testing.T) {
 		{
 			s:    `SELECT sample(-.3,) FROM tbl`,
 			stmt: nil,
-			err:  "error getting function sample: not found",
+			err:  "function sample not found",
 		},
 
 		{

internal/xsql/sqlValidator.go

@@ -22,12 +22,12 @@ import (
 // Validate validate select statement without context.
 // This is the pre-validation. In planner, there will be a more comprehensive validation after binding
 func Validate(stmt *ast.SelectStatement) error {
-	if ast.HasAggFuncs(stmt.Condition) {
+	if HasAggFuncs(stmt.Condition) {
 		return fmt.Errorf("Not allowed to call aggregate functions in WHERE clause.")
 	}
 
 	for _, d := range stmt.Dimensions {
-		if ast.HasAggFuncs(d.Expr) {
+		if HasAggFuncs(d.Expr) {
 			return fmt.Errorf("Not allowed to call aggregate functions in GROUP BY clause.")
 		}
 	}

internal/xsql/valuer.go

@@ -16,6 +16,7 @@ package xsql
 
 import (
 	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
 	"github.com/lf-edge/ekuiper/pkg/ast"
 	"github.com/lf-edge/ekuiper/pkg/cast"
 	"math"
@@ -322,7 +323,7 @@ func MultiAggregateValuer(data AggregateData, singleCallValuer CallValuer, value
 
 func (a *multiAggregateValuer) Call(name string, args []interface{}) (interface{}, bool) {
 	// assume the aggFuncMap already cache the custom agg funcs in IsAggFunc()
-	isAgg := ast.FuncFinderSingleton().FuncType(name) == ast.AggFunc
+	isAgg := function.IsAggFunc(name)
 	for _, valuer := range a.multiValuer {
 		if a, ok := valuer.(AggregateCallValuer); ok && isAgg {
 			if v, ok := a.Call(name, args); ok {
@@ -405,7 +406,7 @@ func (v *ValuerEval) Eval(expr ast.Expr) interface{} {
 				if len(expr.Args) > 0 {
 					args = make([]interface{}, len(expr.Args))
 					for i, arg := range expr.Args {
-						if aggreValuer, ok := valuer.(AggregateCallValuer); ast.FuncFinderSingleton().IsAggFunc(expr) && ok {
+						if aggreValuer, ok := valuer.(AggregateCallValuer); function.IsAggFunc(expr.Name) && ok {
 							args[i] = aggreValuer.GetAllTuples().AggregateEval(arg, aggreValuer.GetSingleCallValuer())
 						} else {
 							args[i] = v.Eval(arg)

pkg/ast/expr.go

@@ -193,18 +193,6 @@ func (fr *FieldRef) IsAlias() bool {
 	return fr.StreamName == AliasStream
 }
 
-func (fr *FieldRef) IsAggregate() bool {
-	if fr.StreamName != AliasStream {
-		return false
-	}
-	// lazy calculate
-	if fr.isAggregate == nil {
-		tr := IsAggregate(fr.Expression)
-		fr.isAggregate = &tr
-	}
-	return *fr.isAggregate
-}
-
 func (fr *FieldRef) RefSelection(a *AliasRef) {
 	fr.AliasRef = a
 }
@@ -231,7 +219,7 @@ type AliasRef struct {
 	// MUST have after binding, calculate once in initializer. Could be 0 when alias an Expression without col like "1+2"
 	refSources []StreamName
 	// optional, lazy set when calculating isAggregate
-	isAggregate *bool
+	IsAggregate *bool
 }
 
 func NewAliasRef(e Expr) (*AliasRef, error) {

pkg/errorx/errors.go

@@ -14,6 +14,8 @@
 
 package errorx
 
+import "fmt"
+
 type ErrorCode int
 
 const (
@@ -43,3 +45,26 @@ func (e *Error) Error() string {
 func (e *Error) Code() ErrorCode {
 	return e.code
 }
+
+type MultiError map[string]error
+
+func (e MultiError) Error() string {
+	var s string
+	switch len(e) {
+	case 0, 1:
+		s = ""
+	default:
+		s = "Get multiple errors: "
+	}
+	for k, v := range e {
+		s = fmt.Sprintf("%s\n%s:%s", s, k, v.Error())
+	}
+	return s
+}
+
+func (e MultiError) GetError() error {
+	if len(e) > 0 {
+		return e
+	}
+	return nil
+}