feat(portable): runtime support for portable plugins and go sdk

1. Portable plugin runtime support
2. Portable plugin file load
3. Plugin sdk for go

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

feat(portable): REST API for portable plugin management

1. Implement installation (file copy, meta read)
2. Rest endpoint for portable plugin CRD

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

feat(portable): Add test mock in sdk

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

fix(portable): fix some runtime problems

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

fix(portable): restart fail because the old mangos connection is still open

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

test(portable): add unit tests

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

fix(portable): control channel mutex

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

.github/workflows/run_test_case.yaml

@@ -36,6 +36,11 @@ jobs:
             go build -modfile extensions.mod --buildmode=plugin -o plugins/functions/Echo.so extensions/functions/echo/echo.go
             go build -modfile extensions.mod --buildmode=plugin -o plugins/functions/CountPlusOne@v1.0.0.so extensions/functions/countPlusOne/countPlusOne.go
             go build -modfile extensions.mod --buildmode=plugin -o plugins/functions/AccumulateWordCount@v1.0.0.so extensions/functions/accumulateWordCount/accumulateWordCount.go
+            mkdir -p plugins/portable/mirror
+            cd sdk/go/example/mirror
+            go build -o ../../../../plugins/portable/mirror/mirror .
+            cp mirror.json ../../../../plugins/portable/mirror
+            cd ../../../../
             go test --tags="edgex test" ./...
         - uses: actions/upload-artifact@v1
           if: failure()

Makefile

@@ -28,6 +28,7 @@ build_prepare:
 	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/sources
 	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/sinks
 	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/functions
+	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/plugins/portable
 	@mkdir -p $(BUILD_PATH)/$(PACKAGE_NAME)/log
 
 	@cp -r etc/* $(BUILD_PATH)/$(PACKAGE_NAME)/etc

deploy/packages/deb/Makefile

@@ -19,6 +19,7 @@ $(BUILT):
 	rm -rf $(SRCDIR)/plugins/source/*
 	rm -rf $(SRCDIR)/plugins/sinks/*
 	rm -rf $(SRCDIR)/plugins/functions/*
+	rm -rf $(SRCDIR)/plugins/portable/*
 
 clean:
 	rm -rf $(TOPDIR)

deploy/packages/deb/debian/rules

@@ -37,6 +37,7 @@ install: build
 	mkdir -p debian/kuiper/var/lib/kuiper/plugins/sources
 	mkdir -p debian/kuiper/var/lib/kuiper/plugins/sinks
 	mkdir -p debian/kuiper/var/lib/kuiper/plugins/functions
+	mkdir -p debian/kuiper/var/lib/kuiper/plugins/portable
 	mkdir -p debian/kuiper/var/log/kuiper
 	mkdir -p debian/kuiper/etc/kuiper
 	mkdir -p debian/kuiper/lib/systemd/system

deploy/packages/rpm/kuiper.spec

@@ -41,6 +41,7 @@ mkdir -p %{buildroot}%{_var_home}/plugins
 mkdir -p %{buildroot}%{_var_home}/plugins/sources
 mkdir -p %{buildroot}%{_var_home}/plugins/sinks
 mkdir -p %{buildroot}%{_var_home}/plugins/functions
+mkdir -p %{buildroot}%{_var_home}/plugins/portable
 mkdir -p %{buildroot}%{_initddir}
 
 

extensions.mod

@@ -38,6 +38,7 @@ require (
 	github.com/tebeka/strftime v0.1.5 // indirect
 	github.com/ugorji/go/codec v1.2.5
 	github.com/urfave/cli v1.22.0
+	go.nanomsg.org/mangos/v3 v3.2.1
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0

extensions.sum

@@ -6,6 +6,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/sprig/v3 v3.2.1 h1:n6EPaDyLSvCEa3frruQvAiHuNp2dhBlMSmkEr+HuzGc=
 github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
 github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
@@ -54,6 +56,7 @@ github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fxamacker/cbor/v2 v2.2.0 h1:6eXqdDDe588rSYAi1HfZKbx6YYQO4mxQ9eC6xYpU/JQ=
 github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/gdamore/optopia v0.2.0/go.mod h1:YKYEwo5C1Pa617H7NlPcmQXl+vG6YnSSNB44n8dNL0Q=
 github.com/gdexlab/go-render v1.0.1 h1:rxqB3vo5s4n1kF0ySmoNeSPRYkEsyHgln4jFIQY7v0U=
 github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -111,6 +114,7 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR
 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -238,6 +242,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg=
 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
+go.nanomsg.org/mangos/v3 v3.2.1 h1:/7pG6tUJO5ZGznG+waoMy6WrurArODDRJu18848oQnw=
+go.nanomsg.org/mangos/v3 v3.2.1/go.mod h1:RxVwsn46YtfJ74mF8MeVo+MFjg545KCI50NuZrFXmzc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -282,6 +288,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

extensions/sources/random/random.go

@@ -44,7 +44,9 @@ type randomSource struct {
 }
 
 func (s *randomSource) Configure(topic string, props map[string]interface{}) error {
-	cfg := &randomSourceConfig{}
+	cfg := &randomSourceConfig{
+		Format: "json",
+	}
 	err := cast.MapToStruct(props, cfg)
 	if err != nil {
 		return fmt.Errorf("read properties %v fail with error: %v", props, err)
@@ -55,7 +57,7 @@ func (s *randomSource) Configure(topic string, props map[string]interface{}) err
 	if cfg.Pattern == nil {
 		return fmt.Errorf("source `random` property `pattern` is required")
 	}
-	if cfg.Interval <= 0 {
+	if cfg.Seed <= 0 {
 		return fmt.Errorf("source `random` property `seed` must be a positive integer but got %d", cfg.Seed)
 	}
 	if strings.ToLower(cfg.Format) != message.FormatJson {

go.mod

@@ -38,6 +38,7 @@ require (
 	github.com/tebeka/strftime v0.1.5 // indirect
 	github.com/ugorji/go/codec v1.2.5
 	github.com/urfave/cli v1.22.0
+	go.nanomsg.org/mangos/v3 v3.2.1
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0

go.sum

@@ -6,6 +6,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/sprig/v3 v3.2.1 h1:n6EPaDyLSvCEa3frruQvAiHuNp2dhBlMSmkEr+HuzGc=
 github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
 github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
@@ -56,6 +58,7 @@ github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fxamacker/cbor/v2 v2.2.0 h1:6eXqdDDe588rSYAi1HfZKbx6YYQO4mxQ9eC6xYpU/JQ=
 github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/gdamore/optopia v0.2.0/go.mod h1:YKYEwo5C1Pa617H7NlPcmQXl+vG6YnSSNB44n8dNL0Q=
 github.com/gdexlab/go-render v1.0.1 h1:rxqB3vo5s4n1kF0ySmoNeSPRYkEsyHgln4jFIQY7v0U=
 github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -115,6 +118,7 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR
 github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
@@ -243,6 +247,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg=
 github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
+go.nanomsg.org/mangos/v3 v3.2.1 h1:/7pG6tUJO5ZGznG+waoMy6WrurArODDRJu18848oQnw=
+go.nanomsg.org/mangos/v3 v3.2.1/go.mod h1:RxVwsn46YtfJ74mF8MeVo+MFjg545KCI50NuZrFXmzc=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -287,6 +293,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/binder/factory_test.go

@@ -0,0 +1,31 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package binder
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder/mock"
+	"testing"
+)
+
+func TestEntry(t *testing.T) {
+	m := mock.NewMockFactory()
+	e := &FactoryEntry{
+		Name:    "mock",
+		Factory: m,
+	}
+	if e == nil {
+		t.Errorf("cannot instantiate FactoryEntry")
+	}
+}

internal/binder/function/binder_test.go

@@ -0,0 +1,95 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package function
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/mock"
+	"testing"
+)
+
+func TestBinding(t *testing.T) {
+	// Initialize binding
+	m := mock.NewMockFactory()
+	e := binder.FactoryEntry{
+		Name:    "mock",
+		Factory: m,
+	}
+	err := Initialize([]binder.FactoryEntry{e})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	var tests = []struct {
+		name      string
+		isFunc    bool
+		isFuncset bool
+		hasAgg    bool
+		isAgg     bool
+	}{
+		{
+			name:      "mockFunc1",
+			isFunc:    true,
+			isFuncset: true,
+			hasAgg:    false,
+		}, {
+			name:      "mockFunc2",
+			isFunc:    true,
+			isFuncset: true,
+			hasAgg:    false,
+		}, {
+			name:      "count",
+			isFunc:    true,
+			isFuncset: false,
+			hasAgg:    true,
+			isAgg:     true,
+		}, {
+			name:      "echo",
+			isFunc:    false,
+			isFuncset: false,
+			hasAgg:    false,
+		}, {
+			name:      "internal",
+			isFunc:    false,
+			isFuncset: true,
+			hasAgg:    false,
+		}, {
+			name:      "cast",
+			isFunc:    true,
+			isFuncset: false,
+			hasAgg:    true,
+			isAgg:     false,
+		},
+	}
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for _, tt := range tests {
+		_, err := Function(tt.name)
+		isFunc := err == nil
+		if tt.isFunc != isFunc {
+			t.Errorf("%s is function: expect %v but got %v", tt.name, tt.isFunc, isFunc)
+		}
+		isFuncset := HasFunctionSet(tt.name)
+		if tt.isFuncset != isFuncset {
+			t.Errorf("%s is function set: expect %v but got %v", tt.name, tt.isFuncset, isFuncset)
+		}
+		if tt.hasAgg {
+			isAgg := IsAggFunc(tt.name)
+			if tt.isAgg != isAgg {
+				t.Errorf("%s is aggregate: expect %v but got %v", tt.name, tt.isAgg, isAgg)
+			}
+		}
+	}
+}

internal/binder/function/function_test.go

@@ -0,0 +1,70 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package function
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestManager(t *testing.T) {
+	var tests = []struct {
+		name  string
+		found bool
+	}{
+		{
+			name:  "sum",
+			found: true,
+		}, {
+			name:  "agg",
+			found: false,
+		}, {
+			name:  "ln",
+			found: true,
+		}, {
+			name:  "regexp_matches",
+			found: true,
+		}, {
+			name:  "encode",
+			found: true,
+		}, {
+			name:  "json_path_query",
+			found: true,
+		}, {
+			name:  "window_start",
+			found: true,
+		}, {
+			name:  "cardinality",
+			found: true,
+		},
+	}
+	m := GetManager()
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for _, tt := range tests {
+		f, _ := m.Function(tt.name)
+		found := f != nil
+		if tt.found != found {
+			t.Errorf("%s result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", tt.name, tt.found, found)
+		}
+	}
+	h := m.HasFunctionSet("internal")
+	if !h {
+		t.Errorf("can't find function set internal")
+	}
+	h = m.HasFunctionSet("other")
+	if h {
+		t.Errorf("find undefined function set other")
+	}
+}

internal/binder/io/binder_test.go

@@ -0,0 +1,71 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package io
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/mock"
+	"testing"
+)
+
+func TestBindings(t *testing.T) {
+	m := mock.NewMockFactory()
+	e := binder.FactoryEntry{
+		Name:    "mock",
+		Factory: m,
+	}
+	err := Initialize([]binder.FactoryEntry{e})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	var tests = []struct {
+		name     string
+		isSource bool
+		isSink   bool
+	}{
+		{
+			name:     "unknown",
+			isSource: false,
+			isSink:   false,
+		}, {
+			name:     "mqtt",
+			isSource: true,
+			isSink:   true,
+		}, {
+			name:     "mock1",
+			isSource: true,
+			isSink:   true,
+		}, {
+			name:     "rest",
+			isSource: false,
+			isSink:   true,
+		},
+	}
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for _, tt := range tests {
+		_, err := Source(tt.name)
+		isSource := err == nil
+		if tt.isSource != isSource {
+			t.Errorf("%s is source: expect %v but got %v", tt.name, tt.isSource, isSource)
+		}
+		_, err = Sink(tt.name)
+		isSink := err == nil
+		if tt.isSink != isSink {
+			t.Errorf("%s is sink: expect %v but got %v", tt.name, tt.isSink, isSink)
+		}
+	}
+}

internal/binder/meta/bind.go

@@ -21,6 +21,7 @@ import (
 	"github.com/lf-edge/ekuiper/internal/meta"
 )
 
+// Bind Must run after function and io bound
 func Bind() {
 	if err := meta.ReadSourceMetaDir(func(name string) bool {
 		s, _ := io.Source(name)

internal/binder/mock/mock_factory.go

@@ -0,0 +1,106 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"github.com/lf-edge/ekuiper/pkg/errorx"
+	"strings"
+)
+
+type MockFactory struct {
+}
+
+func NewMockFactory() *MockFactory {
+	return &MockFactory{}
+}
+
+func (f *MockFactory) Source(name string) (api.Source, error) {
+	if strings.HasPrefix(name, "mock") {
+		return &mockSource{}, nil
+	} else {
+		return nil, errorx.NotFoundErr
+	}
+}
+
+func (f *MockFactory) Sink(name string) (api.Sink, error) {
+	if strings.HasPrefix(name, "mock") {
+		return &mockSink{}, nil
+	} else {
+		return nil, errorx.NotFoundErr
+	}
+}
+
+func (f *MockFactory) Function(name string) (api.Function, error) {
+	if strings.HasPrefix(name, "mock") {
+		return &mockFunc{}, nil
+	} else {
+		return nil, errorx.NotFoundErr
+	}
+}
+
+func (f *MockFactory) HasFunctionSet(funcName string) bool {
+	if strings.HasPrefix(funcName, "mock") {
+		return true
+	} else {
+		return false
+	}
+}
+
+type mockFunc struct{}
+
+func (m *mockFunc) Validate(_ []interface{}) error {
+	return nil
+}
+
+func (m *mockFunc) Exec(_ []interface{}, _ api.FunctionContext) (interface{}, bool) {
+	return nil, true
+}
+
+func (m *mockFunc) IsAggregate() bool {
+	return false
+}
+
+type mockSource struct{}
+
+func (m *mockSource) Open(_ api.StreamContext, _ chan<- api.SourceTuple, _ chan<- error) {
+	return
+}
+
+func (m *mockSource) Configure(_ string, _ map[string]interface{}) error {
+	return nil
+}
+
+func (m *mockSource) Close(_ api.StreamContext) error {
+	return nil
+}
+
+type mockSink struct{}
+
+func (m *mockSink) Open(_ api.StreamContext) error {
+	return nil
+}
+
+func (m *mockSink) Configure(_ map[string]interface{}) error {
+	return nil
+}
+
+func (m *mockSink) Collect(_ api.StreamContext, _ interface{}) error {
+	return nil
+}
+
+func (m *mockSink) Close(_ api.StreamContext) error {
+	return nil
+}

internal/meta/func_meta.go

@@ -63,10 +63,9 @@ func newUiFuncs(fi *fileFuncs) *uiFuncs {
 	return uis
 }
 
-var gFuncmetadata map[string]*uiFuncs
+var gFuncmetadata = make(map[string]*uiFuncs)
 
 func ReadFuncMetaDir(checker InstallChecker) error {
-	gFuncmetadata = make(map[string]*uiFuncs)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err

internal/meta/sinkMeta.go

@@ -155,10 +155,9 @@ func newUiSink(fi *fileSink) (*uiSink, error) {
 	return ui, err
 }
 
-var gSinkmetadata map[string]*uiSink //immutable
+var gSinkmetadata = make(map[string]*uiSink) //immutable
 
 func ReadSinkMetaDir(checker InstallChecker) error {
-	gSinkmetadata = make(map[string]*uiSink)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err

internal/meta/sourceMeta.go

@@ -62,7 +62,7 @@ func newUiSource(fi *fileSource) (*uiSource, error) {
 	return ui, nil
 }
 
-var gSourceproperty map[string]*sourceProperty
+var gSourceproperty = make(map[string]*sourceProperty)
 
 func UninstallSource(name string) {
 	if v, ok := gSourceproperty[name+".json"]; ok {
@@ -114,7 +114,6 @@ func ReadSourceMetaFile(filePath string, installed bool) error {
 }
 
 func ReadSourceMetaDir(checker InstallChecker) error {
-	gSourceproperty = make(map[string]*sourceProperty)
 	confDir, err := conf.GetConfLoc()
 	if nil != err {
 		return err

internal/plugin/native/manager.go

@@ -25,10 +25,10 @@ import (
 	"github.com/lf-edge/ekuiper/internal/pkg/filex"
 	"github.com/lf-edge/ekuiper/internal/pkg/httpx"
 	"github.com/lf-edge/ekuiper/internal/pkg/store"
+	plugin2 "github.com/lf-edge/ekuiper/internal/plugin"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/errorx"
 	"github.com/lf-edge/ekuiper/pkg/kv"
-	"github.com/pkg/errors"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -78,8 +78,8 @@ func InitManager() (*Manager, error) {
 		return nil, fmt.Errorf("error when opening db: %v", err)
 	}
 	plugins := make([]map[string]string, 3)
-	for i := range PluginTypes {
-		names, err := findAll(PluginType(i), pluginDir)
+	for i := range plugin2.PluginTypes {
+		names, err := findAll(plugin2.PluginType(i), pluginDir)
 		if err != nil {
 			return nil, fmt.Errorf("fail to find existing plugins: %s", err)
 		}
@@ -87,7 +87,7 @@ func InitManager() (*Manager, error) {
 	}
 	registry := &Manager{plugins: plugins, symbols: make(map[string]string), db: db, pluginDir: pluginDir, etcDir: etcDir, runtime: make(map[string]plugin.Symbol)}
 
-	for pf := range plugins[FUNCTION] {
+	for pf := range plugins[plugin2.FUNCTION] {
 		l := make([]string, 0)
 		if ok, err := db.Get(pf, &l); ok {
 			registry.storeSymbols(pf, l)
@@ -101,9 +101,9 @@ func InitManager() (*Manager, error) {
 	return registry, nil
 }
 
-func findAll(t PluginType, pluginDir string) (result map[string]string, err error) {
+func findAll(t plugin2.PluginType, pluginDir string) (result map[string]string, err error) {
 	result = make(map[string]string)
-	dir := path.Join(pluginDir, PluginTypes[t])
+	dir := path.Join(pluginDir, plugin2.PluginTypes[t])
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		return
@@ -123,7 +123,7 @@ func GetManager() *Manager {
 	return manager
 }
 
-func (rr *Manager) get(t PluginType, name string) (string, bool) {
+func (rr *Manager) get(t plugin2.PluginType, name string) (string, bool) {
 	rr.RLock()
 	result := rr.plugins[t]
 	rr.RUnlock()
@@ -131,7 +131,7 @@ func (rr *Manager) get(t PluginType, name string) (string, bool) {
 	return r, ok
 }
 
-func (rr *Manager) store(t PluginType, name string, version string) {
+func (rr *Manager) store(t plugin2.PluginType, name string, version string) {
 	rr.Lock()
 	rr.plugins[t][name] = version
 	rr.Unlock()
@@ -161,7 +161,7 @@ func (rr *Manager) removeSymbols(symbols []string) {
 
 // API for management
 
-func (rr *Manager) List(t PluginType) []string {
+func (rr *Manager) List(t plugin2.PluginType) []string {
 	rr.RLock()
 	result := rr.plugins[t]
 	rr.RUnlock()
@@ -183,9 +183,9 @@ func (rr *Manager) ListSymbols() []string {
 	return keys
 }
 
-func (rr *Manager) GetPluginVersionBySymbol(t PluginType, symbolName string) (string, bool) {
+func (rr *Manager) GetPluginVersionBySymbol(t plugin2.PluginType, symbolName string) (string, bool) {
 	switch t {
-	case FUNCTION:
+	case plugin2.FUNCTION:
 		rr.RLock()
 		result := rr.plugins[t]
 		name, ok := rr.symbols[symbolName]
@@ -201,9 +201,9 @@ func (rr *Manager) GetPluginVersionBySymbol(t PluginType, symbolName string) (st
 	}
 }
 
-func (rr *Manager) GetPluginBySymbol(t PluginType, symbolName string) (string, bool) {
+func (rr *Manager) GetPluginBySymbol(t plugin2.PluginType, symbolName string) (string, bool) {
 	switch t {
-	case FUNCTION:
+	case plugin2.FUNCTION:
 		rr.RLock()
 		defer rr.RUnlock()
 		name, ok := rr.symbols[symbolName]
@@ -213,7 +213,7 @@ func (rr *Manager) GetPluginBySymbol(t PluginType, symbolName string) (string, b
 	}
 }
 
-func (rr *Manager) Register(t PluginType, j Plugin) error {
+func (rr *Manager) Register(t plugin2.PluginType, j plugin2.Plugin) error {
 	name, uri, shellParas := j.GetName(), j.GetFile(), j.GetShellParas()
 	//Validation
 	name = strings.Trim(name, " ")
@@ -232,7 +232,7 @@ func (rr *Manager) Register(t PluginType, j Plugin) error {
 		}
 	}
 	var err error
-	if t == FUNCTION {
+	if t == plugin2.FUNCTION {
 		if len(j.GetSymbols()) > 0 {
 			err = rr.db.Set(name, j.GetSymbols())
 			if err != nil {
@@ -262,7 +262,7 @@ func (rr *Manager) Register(t PluginType, j Plugin) error {
 		err = rr.db.Set(name, j.GetSymbols())
 	}
 	if err != nil { //Revert for any errors
-		if t == SOURCE && len(unzipFiles) == 1 { //source that only copy so file
+		if t == plugin2.SOURCE && len(unzipFiles) == 1 { //source that only copy so file
 			os.RemoveAll(unzipFiles[0])
 		}
 		if len(j.GetSymbols()) > 0 {
@@ -275,16 +275,16 @@ func (rr *Manager) Register(t PluginType, j Plugin) error {
 	rr.store(t, name, version)
 
 	switch t {
-	case SINK:
-		if err := meta.ReadSinkMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
+	case plugin2.SINK:
+		if err := meta.ReadSinkMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSinkFile:%v", err)
 		}
-	case SOURCE:
-		if err := meta.ReadSourceMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
+	case plugin2.SOURCE:
+		if err := meta.ReadSourceMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSourceFile:%v", err)
 		}
-	case FUNCTION:
-		if err := meta.ReadFuncMetaFile(path.Join(rr.etcDir, PluginTypes[t], name+`.json`), true); nil != err {
+	case plugin2.FUNCTION:
+		if err := meta.ReadFuncMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readFuncFile:%v", err)
 		}
 	}
@@ -311,7 +311,7 @@ func (rr *Manager) RegisterFuncs(name string, functions []string) error {
 	return rr.storeSymbols(name, functions)
 }
 
-func (rr *Manager) Delete(t PluginType, name string, stop bool) error {
+func (rr *Manager) Delete(t plugin2.PluginType, name string, stop bool) error {
 	name = strings.Trim(name, " ")
 	if name == "" {
 		return fmt.Errorf("invalid name %s: should not be empty", name)
@@ -325,19 +325,19 @@ func (rr *Manager) Delete(t PluginType, name string, stop bool) error {
 		soPath,
 	}
 	// Find etc folder
-	etcPath := path.Join(rr.etcDir, PluginTypes[t], name)
+	etcPath := path.Join(rr.etcDir, plugin2.PluginTypes[t], name)
 	if fi, err := os.Stat(etcPath); err == nil {
 		if fi.Mode().IsDir() {
 			paths = append(paths, etcPath)
 		}
 	}
 	switch t {
-	case SOURCE:
-		paths = append(paths, path.Join(rr.etcDir, PluginTypes[t], name+".yaml"))
+	case plugin2.SOURCE:
+		paths = append(paths, path.Join(rr.etcDir, plugin2.PluginTypes[t], name+".yaml"))
 		meta.UninstallSource(name)
-	case SINK:
+	case plugin2.SINK:
 		meta.UninstallSink(name)
-	case FUNCTION:
+	case plugin2.FUNCTION:
 		old := make([]string, 0)
 		if ok, err := rr.db.Get(name, &old); err != nil {
 			return err
@@ -366,7 +366,7 @@ func (rr *Manager) Delete(t PluginType, name string, stop bool) error {
 	}
 
 	if len(results) > 0 {
-		return errors.New(strings.Join(results, "\n"))
+		return fmt.Errorf(strings.Join(results, "\n"))
 	} else {
 		rr.store(t, name, DELETED)
 		if stop {
@@ -378,7 +378,7 @@ func (rr *Manager) Delete(t PluginType, name string, stop bool) error {
 		return nil
 	}
 }
-func (rr *Manager) GetPluginInfo(t PluginType, name string) (map[string]interface{}, bool) {
+func (rr *Manager) GetPluginInfo(t plugin2.PluginType, name string) (map[string]interface{}, bool) {
 	v, ok := rr.get(t, name)
 	if strings.HasPrefix(v, "v") {
 		v = v[1:]
@@ -388,7 +388,7 @@ func (rr *Manager) GetPluginInfo(t PluginType, name string) (map[string]interfac
 			"name":    name,
 			"version": v,
 		}
-		if t == FUNCTION {
+		if t == plugin2.FUNCTION {
 			l := make([]string, 0)
 			if ok, _ := rr.db.Get(name, &l); ok {
 				r["functions"] = l
@@ -400,9 +400,9 @@ func (rr *Manager) GetPluginInfo(t PluginType, name string) (map[string]interfac
 	return nil, false
 }
 
-func (rr *Manager) install(t PluginType, name, src string, shellParas []string) ([]string, string, error) {
+func (rr *Manager) install(t plugin2.PluginType, name, src string, shellParas []string) ([]string, string, error) {
 	var filenames []string
-	var tempPath = path.Join(rr.pluginDir, "temp", PluginTypes[t], name)
+	var tempPath = path.Join(rr.pluginDir, "temp", plugin2.PluginTypes[t], name)
 	defer os.RemoveAll(tempPath)
 	r, err := zip.OpenReader(src)
 	if err != nil {
@@ -413,9 +413,9 @@ func (rr *Manager) install(t PluginType, name, src string, shellParas []string)
 	soPrefix := regexp.MustCompile(fmt.Sprintf(`^((%s)|(%s))(@.*)?\.so$`, name, ucFirst(name)))
 	var yamlFile, yamlPath, version string
 	expFiles := 1
-	if t == SOURCE {
+	if t == plugin2.SOURCE {
 		yamlFile = name + ".yaml"
-		yamlPath = path.Join(rr.etcDir, PluginTypes[t], yamlFile)
+		yamlPath = path.Join(rr.etcDir, plugin2.PluginTypes[t], yamlFile)
 		expFiles = 2
 	}
 	var revokeFiles []string
@@ -430,14 +430,14 @@ func (rr *Manager) install(t PluginType, name, src string, shellParas []string)
 			revokeFiles = append(revokeFiles, yamlPath)
 			filenames = append(filenames, yamlPath)
 		} else if fileName == name+".json" {
-			jsonPath := path.Join(rr.etcDir, PluginTypes[t], fileName)
+			jsonPath := path.Join(rr.etcDir, plugin2.PluginTypes[t], fileName)
 			if err := filex.UnzipTo(file, jsonPath); nil != err {
 				conf.Log.Errorf("Failed to decompress the metadata %s file", fileName)
 			} else {
 				revokeFiles = append(revokeFiles, jsonPath)
 			}
 		} else if soPrefix.Match([]byte(fileName)) {
-			soPath := path.Join(rr.pluginDir, PluginTypes[t], fileName)
+			soPath := path.Join(rr.pluginDir, plugin2.PluginTypes[t], fileName)
 			err = filex.UnzipTo(file, soPath)
 			if err != nil {
 				return filenames, "", err
@@ -446,7 +446,7 @@ func (rr *Manager) install(t PluginType, name, src string, shellParas []string)
 			revokeFiles = append(revokeFiles, soPath)
 			_, version = parseName(fileName)
 		} else if strings.HasPrefix(fileName, "etc/") {
-			err = filex.UnzipTo(file, path.Join(rr.etcDir, PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
+			err = filex.UnzipTo(file, path.Join(rr.etcDir, plugin2.PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
 			if err != nil {
 				return filenames, "", err
 			}
@@ -484,7 +484,7 @@ func (rr *Manager) install(t PluginType, name, src string, shellParas []string)
 			return filenames, "", err
 		} else {
 			conf.Log.Infof(`run install script:%s`, outb.String())
-			conf.Log.Infof("install %s plugin %s", PluginTypes[t], name)
+			conf.Log.Infof("install %s plugin %s", plugin2.PluginTypes[t], name)
 		}
 	}
 	return filenames, version, nil
@@ -493,7 +493,7 @@ func (rr *Manager) install(t PluginType, name, src string, shellParas []string)
 // binder factory implementations
 
 func (rr *Manager) Source(name string) (api.Source, error) {
-	nf, err := rr.loadRuntime(SOURCE, name)
+	nf, err := rr.loadRuntime(plugin2.SOURCE, name)
 	if err != nil {
 		return nil, err
 	}
@@ -511,7 +511,7 @@ func (rr *Manager) Source(name string) (api.Source, error) {
 }
 
 func (rr *Manager) Sink(name string) (api.Sink, error) {
-	nf, err := rr.loadRuntime(SINK, name)
+	nf, err := rr.loadRuntime(plugin2.SINK, name)
 	if err != nil {
 		return nil, err
 	}
@@ -531,7 +531,7 @@ func (rr *Manager) Sink(name string) (api.Sink, error) {
 }
 
 func (rr *Manager) Function(name string) (api.Function, error) {
-	nf, err := rr.loadRuntime(FUNCTION, name)
+	nf, err := rr.loadRuntime(plugin2.FUNCTION, name)
 	if err != nil {
 		return nil, err
 	}
@@ -551,14 +551,14 @@ func (rr *Manager) Function(name string) (api.Function, error) {
 }
 
 func (rr *Manager) HasFunctionSet(name string) bool {
-	_, ok := rr.get(FUNCTION, name)
+	_, ok := rr.get(plugin2.FUNCTION, name)
 	return ok
 }
 
 // If not found, return nil,nil; Other errors return nil, err
-func (rr *Manager) loadRuntime(t PluginType, name string) (plugin.Symbol, error) {
+func (rr *Manager) loadRuntime(t plugin2.PluginType, name string) (plugin.Symbol, error) {
 	ut := ucFirst(name)
-	ptype := PluginTypes[t]
+	ptype := plugin2.PluginTypes[t]
 	key := ptype + "/" + name
 	var nf plugin.Symbol
 	rr.RLock()
@@ -590,7 +590,7 @@ func (rr *Manager) loadRuntime(t PluginType, name string) (plugin.Symbol, error)
 }
 
 // Return the lowercase version of so name. It may be upper case in path.
-func (rr *Manager) getSoFilePath(t PluginType, name string, isSoName bool) (string, error) {
+func (rr *Manager) getSoFilePath(t plugin2.PluginType, name string, isSoName bool) (string, error) {
 	var (
 		v      string
 		soname string
@@ -614,9 +614,9 @@ func (rr *Manager) getSoFilePath(t PluginType, name string, isSoName bool) (stri
 	if v != "" {
 		soFile = fmt.Sprintf("%s@%s.so", soname, v)
 	}
-	p := path.Join(rr.pluginDir, PluginTypes[t], soFile)
+	p := path.Join(rr.pluginDir, plugin2.PluginTypes[t], soFile)
 	if _, err := os.Stat(p); err != nil {
-		p = path.Join(rr.pluginDir, PluginTypes[t], ucFirst(soFile))
+		p = path.Join(rr.pluginDir, plugin2.PluginTypes[t], ucFirst(soFile))
 	}
 	if _, err := os.Stat(p); err != nil {
 		return "", errorx.NewWithCode(errorx.NOT_FOUND, fmt.Sprintf("cannot find .so file for plugin %s", soname))

internal/plugin/native/manager_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"github.com/lf-edge/ekuiper/internal/binder"
 	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/internal/plugin"
 	"github.com/lf-edge/ekuiper/internal/testx"
 	"net/http"
 	"net/http/httptest"
@@ -49,7 +50,7 @@ func TestManager_Register(t *testing.T) {
 	endpoint := s.URL
 
 	data := []struct {
-		t       PluginType
+		t       plugin.PluginType
 		n       string
 		u       string
 		v       string
@@ -58,62 +59,62 @@ func TestManager_Register(t *testing.T) {
 		err     error
 	}{
 		{
-			t:   SOURCE,
+			t:   plugin.SOURCE,
 			n:   "",
 			u:   "",
 			err: errors.New("invalid name : should not be empty"),
 		}, {
-			t:   SOURCE,
+			t:   plugin.SOURCE,
 			n:   "zipMissConf",
 			u:   endpoint + "/sources/zipMissConf.zip",
 			err: errors.New("fail to install plugin: invalid zip file: so file or conf file is missing"),
 		}, {
-			t:   SINK,
+			t:   plugin.SINK,
 			n:   "urlerror",
 			u:   endpoint + "/sinks/nozip",
 			err: errors.New("invalid uri " + endpoint + "/sinks/nozip"),
 		}, {
-			t:   SINK,
+			t:   plugin.SINK,
 			n:   "zipWrongname",
 			u:   endpoint + "/sinks/zipWrongName.zip",
 			err: errors.New("fail to install plugin: invalid zip file: so file or conf file is missing"),
 		}, {
-			t:   FUNCTION,
+			t:   plugin.FUNCTION,
 			n:   "zipMissSo",
 			u:   endpoint + "/functions/zipMissSo.zip",
 			err: errors.New("fail to install plugin: invalid zip file: so file or conf file is missing"),
 		}, {
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random2",
 			u: endpoint + "/sources/random2.zip",
 		}, {
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random3",
 			u: endpoint + "/sources/random3.zip",
 			v: "1.0.0",
 		}, {
-			t:       SINK,
+			t:       plugin.SINK,
 			n:       "file2",
 			u:       endpoint + "/sinks/file2.zip",
 			lowerSo: true,
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			n: "echo2",
 			u: endpoint + "/functions/echo2.zip",
 			f: []string{"echo2", "echo3"},
 		}, {
-			t:   FUNCTION,
+			t:   plugin.FUNCTION,
 			n:   "echo2",
 			u:   endpoint + "/functions/echo2.zip",
 			err: errors.New("invalid name echo2: duplicate"),
 		}, {
-			t:   FUNCTION,
+			t:   plugin.FUNCTION,
 			n:   "misc",
 			u:   endpoint + "/functions/echo2.zip",
 			f:   []string{"misc", "echo3"},
 			err: errors.New("function name echo3 already exists"),
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			n: "comp",
 			u: endpoint + "/functions/comp.zip",
 		},
@@ -121,17 +122,17 @@ func TestManager_Register(t *testing.T) {
 
 	fmt.Printf("The test bucket size is %d.\n\n", len(data))
 	for i, tt := range data {
-		var p Plugin
-		if tt.t == FUNCTION {
-			p = &FuncPlugin{
-				IOPlugin: IOPlugin{
+		var p plugin.Plugin
+		if tt.t == plugin.FUNCTION {
+			p = &plugin.FuncPlugin{
+				IOPlugin: plugin.IOPlugin{
 					Name: tt.n,
 					File: tt.u,
 				},
 				Functions: tt.f,
 			}
 		} else {
-			p = &IOPlugin{
+			p = &plugin.IOPlugin{
 				Name: tt.n,
 				File: tt.u,
 			}
@@ -151,17 +152,17 @@ func TestManager_Register(t *testing.T) {
 
 func TestManager_List(t *testing.T) {
 	data := []struct {
-		t PluginType
+		t plugin.PluginType
 		r []string
 	}{
 		{
-			t: SOURCE,
+			t: plugin.SOURCE,
 			r: []string{"random", "random2", "random3"},
 		}, {
-			t: SINK,
+			t: plugin.SINK,
 			r: []string{"file", "file2"},
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			r: []string{"accumulateWordCount", "comp", "countPlusOne", "echo", "echo2"},
 		},
 	}
@@ -183,7 +184,7 @@ func TestManager_Symbols(t *testing.T) {
 	if !reflect.DeepEqual(r, result) {
 		t.Errorf("result mismatch:\n  exp=%v\n  got=%v\n\n", r, result)
 	}
-	p, ok := manager.GetPluginBySymbol(FUNCTION, "echo3")
+	p, ok := manager.GetPluginBySymbol(plugin.FUNCTION, "echo3")
 	if !ok {
 		t.Errorf("cannot find echo3 symbol")
 	}
@@ -194,26 +195,26 @@ func TestManager_Symbols(t *testing.T) {
 
 func TestManager_Desc(t *testing.T) {
 	data := []struct {
-		t PluginType
+		t plugin.PluginType
 		n string
 		r map[string]interface{}
 	}{
 		{
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random2",
 			r: map[string]interface{}{
 				"name":    "random2",
 				"version": "",
 			},
 		}, {
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random3",
 			r: map[string]interface{}{
 				"name":    "random3",
 				"version": "1.0.0",
 			},
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			n: "echo2",
 			r: map[string]interface{}{
 				"name":      "echo2",
@@ -238,24 +239,24 @@ func TestManager_Desc(t *testing.T) {
 
 func TestManager_Delete(t *testing.T) {
 	data := []struct {
-		t   PluginType
+		t   plugin.PluginType
 		n   string
 		err error
 	}{
 		{
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random2",
 		}, {
-			t: SINK,
+			t: plugin.SINK,
 			n: "file2",
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			n: "echo2",
 		}, {
-			t: SOURCE,
+			t: plugin.SOURCE,
 			n: "random3",
 		}, {
-			t: FUNCTION,
+			t: plugin.FUNCTION,
 			n: "comp",
 		},
 	}
@@ -269,7 +270,7 @@ func TestManager_Delete(t *testing.T) {
 	}
 }
 
-func checkFile(pluginDir string, etcDir string, t PluginType, name string, version string, lowerSo bool) error {
+func checkFile(pluginDir string, etcDir string, t plugin.PluginType, name string, version string, lowerSo bool) error {
 	var soName string
 	if !lowerSo {
 		soName = ucFirst(name) + ".so"
@@ -283,13 +284,13 @@ func checkFile(pluginDir string, etcDir string, t PluginType, name string, versi
 		}
 	}
 
-	soPath := path.Join(pluginDir, PluginTypes[t], soName)
+	soPath := path.Join(pluginDir, plugin.PluginTypes[t], soName)
 	_, err := os.Stat(soPath)
 	if err != nil {
 		return err
 	}
-	if t == SOURCE {
-		etcPath := path.Join(etcDir, PluginTypes[t], name+".yaml")
+	if t == plugin.SOURCE {
+		etcPath := path.Join(etcDir, plugin.PluginTypes[t], name+".yaml")
 		_, err = os.Stat(etcPath)
 		if err != nil {
 			return err

internal/plugin/native/plugin.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package native
+package plugin
 
 type PluginType int
 
@@ -20,6 +20,7 @@ const (
 	SOURCE PluginType = iota
 	SINK
 	FUNCTION
+	PORTABLE
 )
 
 var PluginTypes = []string{"sources", "sinks", "functions"}

internal/plugin/plugin_test.go

@@ -0,0 +1,76 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package plugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestDecode(t *testing.T) {
+	var tests = []struct {
+		t          PluginType
+		j          string
+		name       string
+		file       string
+		shellParas []string
+		symbols    []string
+	}{
+		{
+			t:    PORTABLE,
+			j:    "{\"name\":\"mirror\",\"file\":\"mirror_win.zip\"}",
+			name: "mirror",
+			file: "mirror_win.zip",
+		}, {
+			t:          SINK,
+			j:          `{"name":"tdengine","file":"https://packages.emqx.io/kuiper-plugins/1.3.1/debian/sinks/tdengine_amd64.zip","shellParas": ["2.0.3.1"]}`,
+			name:       "tdengine",
+			file:       "https://packages.emqx.io/kuiper-plugins/1.3.1/debian/sinks/tdengine_amd64.zip",
+			shellParas: []string{"2.0.3.1"},
+		}, {
+			t:       FUNCTION,
+			j:       `{"name":"image","file":"https://packages.emqx.io/kuiper-plugins/1.3.1/debian/functions/image_amd64.zip","functions": ["resize","thumbnail"]}`,
+			name:    "image",
+			file:    "https://packages.emqx.io/kuiper-plugins/1.3.1/debian/functions/image_amd64.zip",
+			symbols: []string{"resize", "thumbnail"},
+		}, {
+			t: FUNCTION,
+			j: "{\"name1\":\"image\",\"file1\":\"abc\"}",
+		},
+	}
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		p := NewPluginByType(tt.t)
+		err := json.Unmarshal([]byte(tt.j), p)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if tt.name != p.GetName() {
+			t.Errorf("%d name mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.name, p.GetName())
+		}
+		if tt.file != p.GetFile() {
+			t.Errorf("%d file mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.file, p.GetFile())
+		}
+		if !reflect.DeepEqual(tt.shellParas, p.GetShellParas()) {
+			t.Errorf("%d shellParas mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.shellParas, p.GetShellParas())
+		}
+		if !reflect.DeepEqual(tt.symbols, p.GetSymbols()) {
+			t.Errorf("%d symbols mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.symbols, p.GetSymbols())
+		}
+	}
+}

internal/plugin/portable/factory.go

@@ -0,0 +1,73 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"sync"
+)
+
+func (m *Manager) Source(name string) (api.Source, error) {
+	meta, ok := m.GetPluginMeta(plugin.SOURCE, name)
+	if !ok {
+		return nil, nil
+	}
+	return runtime.NewPortableSource(name, meta), nil
+}
+
+func (m *Manager) Sink(name string) (api.Sink, error) {
+	meta, ok := m.GetPluginMeta(plugin.SINK, name)
+	if !ok {
+		return nil, nil
+	}
+	return runtime.NewPortableSink(name, meta), nil
+}
+
+var funcInsMap = &sync.Map{}
+
+func (m *Manager) Function(name string) (api.Function, error) {
+	ins, ok := funcInsMap.Load(name)
+	if ok {
+		return ins.(api.Function), nil
+	}
+	meta, ok := m.GetPluginMeta(plugin.FUNCTION, name)
+	if !ok {
+		return nil, nil
+	}
+	f, err := runtime.NewPortableFunc(name, meta)
+	if err != nil {
+		conf.Log.Errorf("Error creating function %v", err)
+		return nil, err
+	}
+	funcInsMap.Store(name, f)
+	return f, nil
+}
+
+func (m *Manager) HasFunctionSet(funcName string) bool {
+	_, ok := m.reg.GetSymbol(plugin.FUNCTION, funcName)
+	return ok
+}
+
+// Clean up function map
+func (m *Manager) Clean() {
+	funcInsMap.Range(func(_, ins interface{}) bool {
+		f := ins.(*runtime.PortableFunc)
+		_ = f.Close()
+		return true
+	})
+}

internal/plugin/portable/manager.go

@@ -0,0 +1,348 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/meta"
+	"github.com/lf-edge/ekuiper/internal/pkg/filex"
+	"github.com/lf-edge/ekuiper/internal/pkg/httpx"
+	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+var manager *Manager
+
+type Manager struct {
+	pluginDir string
+	etcDir    string
+	reg       *registry // can be replaced with kv
+}
+
+// InitManager must only be called once
+func InitManager() (*Manager, error) {
+	pluginDir, err := conf.GetPluginsLoc()
+	if err != nil {
+		return nil, fmt.Errorf("cannot find plugins folder: %s", err)
+	}
+	etcDir, err := conf.GetConfLoc()
+	if err != nil {
+		return nil, fmt.Errorf("cannot find etc folder: %s", err)
+	}
+	registry := &registry{
+		RWMutex:   sync.RWMutex{},
+		plugins:   make(map[string]*PluginInfo),
+		sources:   make(map[string]string),
+		sinks:     make(map[string]string),
+		functions: make(map[string]string),
+	}
+	// Read plugin info from file system
+	pluginDir = filepath.Join(pluginDir, "portable")
+	m := &Manager{
+		pluginDir: pluginDir,
+		etcDir:    etcDir,
+		reg:       registry,
+	}
+	err = m.syncRegistry()
+	if err != nil {
+		return nil, err
+	}
+	manager = m
+	return m, nil
+}
+
+func GetManager() *Manager {
+	return manager
+}
+
+func (m *Manager) syncRegistry() error {
+	files, err := ioutil.ReadDir(m.pluginDir)
+	if err != nil {
+		return fmt.Errorf("read path '%s' error: %v", m.pluginDir, err)
+	}
+	for _, file := range files {
+		if file.IsDir() {
+			err := m.parsePlugin(file.Name())
+			if err != nil {
+				conf.Log.Warn(err)
+			}
+		} else {
+			conf.Log.Warnf("find file `%s`, portable plugin must be a directory", file.Name())
+		}
+	}
+	return nil
+}
+
+func (m *Manager) parsePlugin(name string) error {
+	pi, err := m.parsePluginJson(name)
+	if err != nil {
+		return err
+	}
+	return m.doRegister(name, pi, true)
+}
+
+func (m *Manager) doRegister(name string, pi *PluginInfo, isInit bool) error {
+	exeAbs := filepath.Clean(filepath.Join(m.pluginDir, name, pi.Executable))
+	if _, err := os.Stat(exeAbs); err != nil {
+		return fmt.Errorf("cannot find executable `%s` when loading portable plugins: %v", exeAbs, err)
+	}
+	pi.Executable = exeAbs
+	m.reg.Set(name, pi)
+
+	if !isInit {
+		for _, s := range pi.Sources {
+			if err := meta.ReadSourceMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+`.json`), true); nil != err {
+				conf.Log.Errorf("read source json file:%v", err)
+			}
+		}
+		for _, s := range pi.Sinks {
+			if err := meta.ReadSinkMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.SINK], s+`.json`), true); nil != err {
+				conf.Log.Errorf("read sink json file:%v", err)
+			}
+		}
+		for _, s := range pi.Functions {
+			if err := meta.ReadFuncMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.FUNCTION], s+`.json`), true); nil != err {
+				conf.Log.Errorf("read function json file:%v", err)
+			}
+		}
+	}
+
+	conf.Log.Infof("Installed portable plugin %s successfully", name)
+	return nil
+}
+
+func (m *Manager) parsePluginJson(name string) (*PluginInfo, error) {
+	jsonPath := filepath.Join(m.pluginDir, name, name+".json")
+	pi := &PluginInfo{PluginMeta: runtime.PluginMeta{Name: name}}
+	err := filex.ReadJsonUnmarshal(jsonPath, pi)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read json file `%s` when loading portable plugins: %v", jsonPath, err)
+	}
+	if err := pi.Validate(name); err != nil {
+		return nil, err
+	}
+	if _, ok := m.reg.Get(pi.Name); ok {
+		return nil, fmt.Errorf("portable plugin %s already exists", pi.Name)
+	}
+	return pi, nil
+}
+
+func (m *Manager) Register(p plugin.Plugin) error {
+	name, uri, shellParas := p.GetName(), p.GetFile(), p.GetShellParas()
+	name = strings.Trim(name, " ")
+	if name == "" {
+		return fmt.Errorf("invalid name %s: should not be empty", name)
+	}
+	if !httpx.IsValidUrl(uri) || !strings.HasSuffix(uri, ".zip") {
+		return fmt.Errorf("invalid uri %s", uri)
+	}
+
+	if _, ok := m.reg.Get(name); ok {
+		return fmt.Errorf("invalid name %s: duplicate", name)
+	}
+
+	zipPath := path.Join(m.pluginDir, name+".zip")
+	//clean up: delete zip file and unzip files in error
+	defer os.Remove(zipPath)
+	//download
+	err := httpx.DownloadFile(zipPath, uri)
+	if err != nil {
+		return fmt.Errorf("fail to download file %s: %s", uri, err)
+	}
+	//unzip and copy to destination
+	err = m.install(name, zipPath, shellParas)
+	if err != nil { //Revert for any errors
+		return fmt.Errorf("fail to install plugin: %s", err)
+	}
+	return nil
+}
+
+func (m *Manager) install(name, src string, shellParas []string) (resultErr error) {
+	var (
+		jsonName     = name + ".json"
+		pluginTarget = filepath.Join(m.pluginDir, name)
+		// The map of install files. Used to check if all required files are installed and for reverting
+		installedMap  = make(map[string]string)
+		requiredFiles = []string{jsonName}
+	)
+	defer func() {
+		// remove all installed files if err happens
+		if resultErr != nil {
+			for _, p := range installedMap {
+				_ = os.Remove(p)
+			}
+			_ = os.Remove(pluginTarget)
+		}
+	}()
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	var pi *PluginInfo
+	// Parse json file
+	for _, file := range r.File {
+		if file.Name == jsonName {
+			jf, err := file.Open()
+			if err != nil {
+				err = fmt.Errorf("invalid json file %s: %s", jsonName, err)
+				return err
+			}
+			pi = &PluginInfo{PluginMeta: runtime.PluginMeta{Name: name}}
+			allBytes, err := ioutil.ReadAll(jf)
+			if err != nil {
+				return err
+			}
+			err = json.Unmarshal(allBytes, pi)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if pi == nil {
+		return fmt.Errorf("missing or invalid json file %s", jsonName)
+	}
+	if err = pi.Validate(name); err != nil {
+		return err
+	}
+	if _, ok := m.reg.Get(pi.Name); ok {
+		return fmt.Errorf("portable plugin %s already exists", pi.Name)
+	}
+
+	requiredFiles = append(requiredFiles, pi.Executable)
+	for _, src := range pi.Sources {
+		requiredFiles = append(requiredFiles, fmt.Sprintf("sources/%s.yaml", src))
+	}
+
+	// file copying
+	d := filepath.Clean(pluginTarget)
+	if _, err := os.Stat(d); os.IsNotExist(err) {
+		err = os.MkdirAll(d, 0755)
+		if err != nil {
+			return err
+		}
+	}
+
+	needInstall := false
+	target := ""
+	for _, file := range r.File {
+		fileName := file.Name
+		if strings.HasPrefix(fileName, "sources/") || strings.HasPrefix(fileName, "sinks/") || strings.HasPrefix(fileName, "functions/") {
+			target = path.Join(m.etcDir, fileName)
+		} else {
+			target = path.Join(pluginTarget, fileName)
+			if fileName == "install.sh" {
+				needInstall = true
+			}
+		}
+		err = filex.UnzipTo(file, target)
+		if err != nil {
+			return err
+		}
+		if !file.FileInfo().IsDir() {
+			installedMap[fileName] = target
+		}
+	}
+
+	// Check if all files installed
+	for _, rf := range requiredFiles {
+		if _, ok := installedMap[rf]; !ok {
+			return fmt.Errorf("missing %s", rf)
+		}
+	}
+
+	if needInstall {
+		//run install script if there is
+		spath := path.Join(pluginTarget, "install.sh")
+		shellParas = append(shellParas, spath)
+		if 1 != len(shellParas) {
+			copy(shellParas[1:], shellParas[0:])
+			shellParas[0] = spath
+		}
+		cmd := exec.Command("/bin/sh", shellParas...)
+		var outb, errb bytes.Buffer
+		cmd.Stdout = &outb
+		cmd.Stderr = &errb
+		err = cmd.Run()
+		if err != nil {
+			return err
+		}
+	}
+	return m.doRegister(name, pi, false)
+}
+
+func (m *Manager) List() []*PluginInfo {
+	return m.reg.List()
+}
+
+func (m *Manager) GetPluginMeta(pt plugin.PluginType, symbolName string) (*runtime.PluginMeta, bool) {
+	pname, ok := m.reg.GetSymbol(pt, symbolName)
+	if !ok {
+		return nil, false
+	}
+	pinfo, ok := m.reg.Get(pname)
+	if !ok {
+		return nil, false
+	}
+	return &pinfo.PluginMeta, true
+}
+
+func (m *Manager) GetPluginInfo(pluginName string) (*PluginInfo, bool) {
+	pinfo, ok := m.reg.Get(pluginName)
+	if !ok {
+		return nil, false
+	}
+	return pinfo, true
+}
+
+func (m *Manager) Delete(name string) error {
+	pinfo, ok := m.reg.Get(name)
+	if !ok {
+		return fmt.Errorf("portable plugin %s is not found", name)
+	}
+	// unregister the plugin
+	m.reg.Delete(name)
+	// delete files and uninstall metas
+	for _, s := range pinfo.Sources {
+		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+".yaml")
+		os.Remove(p)
+		p = path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+".json")
+		os.Remove(p)
+		meta.UninstallSource(s)
+	}
+	for _, s := range pinfo.Sinks {
+		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.SINK], s+".json")
+		os.Remove(p)
+		meta.UninstallSink(s)
+	}
+	for _, s := range pinfo.Functions {
+		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.FUNCTION], s+".json")
+		os.Remove(p)
+		meta.UninstallFunc(s)
+	}
+	_ = os.RemoveAll(path.Join(m.pluginDir, name))
+	return nil
+}

internal/plugin/portable/manager_test.go

@@ -0,0 +1,191 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"errors"
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"testing"
+)
+
+// Test only install API. Install from file is tested in the integration test in test/portable_rule_test
+
+func init() {
+	InitManager()
+}
+
+func TestManager_Install(t *testing.T) {
+	s := httptest.NewServer(
+		http.FileServer(http.Dir("../testzips")),
+	)
+	defer s.Close()
+	endpoint := s.URL
+
+	data := []struct {
+		n   string
+		u   string
+		v   string
+		err error
+	}{
+		{ // 0
+			n:   "",
+			u:   "",
+			err: errors.New("invalid name : should not be empty"),
+		}, { // 1
+			n:   "zipMissJson",
+			u:   endpoint + "/functions/misc.zip",
+			err: errors.New("fail to install plugin: missing or invalid json file zipMissJson.json"),
+		}, { // 2
+			n:   "urlerror",
+			u:   endpoint + "/sinks/nozip",
+			err: errors.New("invalid uri " + endpoint + "/sinks/nozip"),
+		}, { // 3
+			n:   "wrong",
+			u:   endpoint + "/portables/wrong.zip",
+			err: errors.New("fail to install plugin: missing mirror.exe"),
+		}, { // 4
+			n:   "wrongname",
+			u:   endpoint + "/portables/mirror.zip",
+			err: errors.New("fail to install plugin: missing or invalid json file wrongname.json"),
+		}, { // 5
+			n: "mirror2",
+			u: endpoint + "/portables/mirror.zip",
+		},
+	}
+
+	fmt.Printf("The test bucket size is %d.\n\n", len(data))
+	for i, tt := range data {
+		p := &plugin.IOPlugin{
+			Name: tt.n,
+			File: tt.u,
+		}
+		err := manager.Register(p)
+		if !reflect.DeepEqual(tt.err, err) {
+			t.Errorf("%d: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
+		} else if tt.err == nil {
+			err := checkFileForMirror(manager.pluginDir, manager.etcDir, true)
+			if err != nil {
+				t.Errorf("%d: error : %s\n\n", i, err)
+			}
+		}
+	}
+}
+
+func TestManager_Read(t *testing.T) {
+	expPlugins := []*PluginInfo{
+		{
+			PluginMeta: runtime.PluginMeta{
+				Name:       "mirror2",
+				Version:    "v1.0.0",
+				Language:   "go",
+				Executable: filepath.Clean(path.Join(manager.pluginDir, "mirror2", "mirror2")),
+			},
+			Sources:   []string{"randomGo"},
+			Sinks:     []string{"fileGo"},
+			Functions: []string{"echoGo"},
+		},
+	}
+	result := manager.List()
+	if len(result) != 2 {
+		t.Errorf("list result mismatch:\n  exp=%v\n  got=%v\n\n", expPlugins, result)
+	}
+
+	_, ok := manager.GetPluginInfo("mirror3")
+	if ok {
+		t.Error("find inexist plugin mirror3")
+	}
+	pi, ok := manager.GetPluginInfo("mirror2")
+	if !ok {
+		t.Error("can't find plugin mirror2")
+	}
+	if !reflect.DeepEqual(expPlugins[0], pi) {
+		t.Errorf("Get plugin mirror2 mismatch:\n exp=%v\n got=%v", expPlugins[0], pi)
+	}
+	_, ok = manager.GetPluginMeta(plugin.SOURCE, "echoGo")
+	if ok {
+		t.Error("find inexist source symbol echo")
+	}
+	m, ok := manager.GetPluginMeta(plugin.SINK, "fileGo")
+	if !ok {
+		t.Error("can't find sink symbol fileGo")
+	}
+	if !reflect.DeepEqual(&(expPlugins[0].PluginMeta), m) {
+		t.Errorf("Get sink symbol mismatch:\n exp=%v\n got=%v", expPlugins[0].PluginMeta, m)
+	}
+}
+
+// This will start channel, so test it in integration tests.
+//func TestFactory(t *testing.T){
+//	_, err := manager.Source("alss")
+//	expErr := fmt.Errorf("can't find random")
+//	if !reflect.DeepEqual(expErr, err){
+//		t.Errorf("error mismatch:\n  exp=%s\n  got=%s\n\n", expErr, err)
+//	}
+//	src, _ := manager.Source("randomGo")
+//	if src  == nil {
+//		t.Errorf("can't get source randomGo")
+//	}
+//	snk, _ := manager.Sink("fileGo")
+//	if snk == nil {
+//		t.Errorf("can't get sink fileGo")
+//	}
+//	fun, _ := manager.Function("echoGo")
+//	if fun == nil {
+//		t.Errorf("can't get function echoGo")
+//	}
+//	ok := manager.HasFunctionSet("echoGo")
+//	if !ok {
+//		t.Errorf("can't check function set")
+//	}
+//}
+
+func TestDelete(t *testing.T) {
+	err := manager.Delete("mirror2")
+	if err != nil {
+		t.Errorf("delete plugin error: %v", err)
+	}
+	err = checkFileForMirror(manager.pluginDir, manager.etcDir, false)
+	if err != nil {
+		t.Errorf("error : %s\n\n", err)
+	}
+}
+
+func checkFileForMirror(pluginDir, etcDir string, exist bool) error {
+	requiredFiles := []string{
+		path.Join(pluginDir, "mirror2", "mirror2"),
+		path.Join(pluginDir, "mirror2", "mirror2.json"),
+		path.Join(etcDir, "sources", "randomGo.yaml"),
+		path.Join(etcDir, "sources", "randomGo.json"),
+		path.Join(etcDir, "functions", "echoGo.json"),
+		path.Join(etcDir, "sinks", "fileGo.json"),
+	}
+	for _, file := range requiredFiles {
+		_, err := os.Stat(file)
+		if exist && err != nil {
+			return err
+		} else if !exist && err == nil {
+			return fmt.Errorf("file still exists: %s", file)
+		}
+	}
+	return nil
+}

internal/plugin/portable/model.go

@@ -0,0 +1,52 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+)
+
+type PluginInfo struct {
+	runtime.PluginMeta
+	Sources   []string `json:"sources"`
+	Sinks     []string `json:"sinks"`
+	Functions []string `json:"functions"`
+}
+
+var langMap = map[string]bool{
+	"go":     true,
+	"python": false,
+}
+
+// Validate TODO validate duplication of source, sink and functions
+func (p *PluginInfo) Validate(expectedName string) error {
+	if p.Name != expectedName {
+		return fmt.Errorf("invalid plugin, expect name '%s' but got '%s'", expectedName, p.Name)
+	}
+	if p.Language == "" {
+		return fmt.Errorf("invalid plugin, missing language")
+	}
+	if p.Executable == "" {
+		return fmt.Errorf("invalid plugin, missing executable")
+	}
+	if len(p.Sources)+len(p.Sinks)+len(p.Functions) == 0 {
+		return fmt.Errorf("invalid plugin, must define at lease one source, sink or function")
+	}
+	if l, ok := langMap[p.Language]; !ok || !l {
+		return fmt.Errorf("invalid plugin, language '%s' is not supported", p.Language)
+	}
+	return nil
+}

internal/plugin/portable/model_test.go

@@ -0,0 +1,107 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"github.com/lf-edge/ekuiper/internal/testx"
+	"reflect"
+	"testing"
+)
+
+func TestValidate(t *testing.T) {
+	var tests = []struct {
+		p   *PluginInfo
+		err string
+	}{
+		{
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:       "mirror",
+					Version:    "1.0.0",
+					Language:   "go",
+					Executable: "mirror.exe",
+				},
+			},
+			err: "invalid plugin, must define at lease one source, sink or function",
+		}, {
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:       "wrr",
+					Version:    "1.0.0",
+					Language:   "go",
+					Executable: "mirror.exe",
+				},
+				Sources: []string{"a", "b"},
+			},
+			err: "invalid plugin, expect name 'mirror' but got 'wrr'",
+		}, {
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:       "mirror",
+					Language:   "go",
+					Executable: "mirror.exe",
+				},
+				Sinks: []string{"a", "b"},
+			},
+			err: "",
+		}, {
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:     "mirror",
+					Version:  "1.0.0",
+					Language: "go",
+				},
+				Sources:   []string{"a", "b"},
+				Sinks:     []string{"a", "b"},
+				Functions: []string{"aa"},
+			},
+			err: "invalid plugin, missing executable",
+		}, {
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:       "mirror",
+					Version:    "1.0.0",
+					Executable: "tt",
+				},
+				Sources:   []string{"a", "b"},
+				Sinks:     []string{"a", "b"},
+				Functions: []string{"aa"},
+			},
+			err: "invalid plugin, missing language",
+		}, {
+			p: &PluginInfo{
+				PluginMeta: runtime.PluginMeta{
+					Name:       "mirror",
+					Version:    "1.0.0",
+					Language:   "c",
+					Executable: "tt",
+				},
+				Sources:   []string{"a", "b"},
+				Sinks:     []string{"a", "b"},
+				Functions: []string{"aa"},
+			},
+			err: "invalid plugin, language 'c' is not supported",
+		},
+	}
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		err := tt.p.Validate("mirror")
+		if !reflect.DeepEqual(tt.err, testx.Errstring(err)) {
+			t.Errorf("%d error mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.err, err.Error())
+		}
+	}
+}

internal/plugin/portable/registry.go

@@ -0,0 +1,97 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"github.com/lf-edge/ekuiper/internal/plugin"
+	"sync"
+)
+
+type registry struct {
+	sync.RWMutex
+	plugins map[string]*PluginInfo
+	// mapping from symbol to plugin. Deduced from plugin set.
+	sources   map[string]string
+	sinks     map[string]string
+	functions map[string]string
+}
+
+// Set prerequisite: the pluginInfo must have been validated that the names are valid
+func (r *registry) Set(name string, pi *PluginInfo) {
+	r.Lock()
+	defer r.Unlock()
+	r.plugins[name] = pi
+	for _, s := range pi.Sources {
+		r.sources[s] = name
+	}
+	for _, s := range pi.Sinks {
+		r.sinks[s] = name
+	}
+	for _, s := range pi.Functions {
+		r.functions[s] = name
+	}
+}
+
+func (r *registry) Get(name string) (*PluginInfo, bool) {
+	r.RLock()
+	defer r.RUnlock()
+	result, ok := r.plugins[name]
+	return result, ok
+}
+
+func (r *registry) GetSymbol(pt plugin.PluginType, symbolName string) (string, bool) {
+	switch pt {
+	case plugin.SOURCE:
+		s, ok := r.sources[symbolName]
+		return s, ok
+	case plugin.SINK:
+		s, ok := r.sinks[symbolName]
+		return s, ok
+	case plugin.FUNCTION:
+		s, ok := r.functions[symbolName]
+		return s, ok
+	default:
+		return "", false
+	}
+}
+
+func (r *registry) List() []*PluginInfo {
+	r.RLock()
+	defer r.RUnlock()
+	var result []*PluginInfo
+	for _, v := range r.plugins {
+		result = append(result, v)
+	}
+	return result
+}
+
+func (r *registry) Delete(name string) {
+	r.Lock()
+	defer r.Unlock()
+	pi, ok := r.plugins[name]
+	if !ok {
+		return
+	}
+	delete(r.plugins, name)
+	for _, s := range pi.Sources {
+		delete(r.sources, s)
+	}
+	for _, s := range pi.Sinks {
+		delete(r.sinks, s)
+	}
+	for _, s := range pi.Functions {
+		delete(r.functions, s)
+	}
+}

internal/plugin/portable/registry_test.go

@@ -0,0 +1,143 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package portable
+
+import (
+	"github.com/lf-edge/ekuiper/internal/plugin"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"reflect"
+	"sync"
+	"testing"
+)
+
+func TestConcurrent(t *testing.T) {
+	r := &registry{
+		RWMutex:   sync.RWMutex{},
+		plugins:   make(map[string]*PluginInfo),
+		sources:   make(map[string]string),
+		sinks:     make(map[string]string),
+		functions: make(map[string]string),
+	}
+	allPlugins := []*PluginInfo{
+		{
+			PluginMeta: runtime.PluginMeta{
+				Name:       "mirror",
+				Version:    "1.3.0",
+				Language:   "go",
+				Executable: "mirror",
+			},
+			Sources:   []string{"random"},
+			Sinks:     []string{"file"},
+			Functions: []string{"echo"},
+		}, {
+			PluginMeta: runtime.PluginMeta{
+				Name:       "next",
+				Version:    "1.3.0",
+				Language:   "python",
+				Executable: "next",
+			},
+			Sinks: []string{"udp", "follower"},
+		}, {
+			PluginMeta: runtime.PluginMeta{
+				Name:       "dummy",
+				Version:    "v0.2",
+				Language:   "go",
+				Executable: "dummy",
+			},
+			Sources:   []string{"new", "can"},
+			Functions: []string{"abc"},
+		},
+	}
+	expectedPlugins := map[string]*PluginInfo{
+		"mirror": allPlugins[0],
+		"next":   allPlugins[1], "dummy": allPlugins[2],
+	}
+
+	expectedSources := map[string]string{
+		"can": "dummy", "new": "dummy", "random": "mirror",
+	}
+	expectedFunctions := map[string]string{
+		"abc": "dummy", "echo": "mirror",
+	}
+	expectedSinks := map[string]string{
+		"file": "mirror", "follower": "next", "udp": "next",
+	}
+	// set concurrently
+	var wg sync.WaitGroup
+	for n, pi := range expectedPlugins {
+		wg.Add(1)
+		go func(name string, pluginInfo *PluginInfo) {
+			defer wg.Done()
+			r.Set(name, pluginInfo)
+		}(n, pi)
+	}
+	wg.Wait()
+
+	if !reflect.DeepEqual(expectedPlugins, r.plugins) {
+		t.Errorf("plugins mismatch: expected %v, got %v", expectedPlugins, r.plugins)
+		return
+	}
+	result := r.List()
+	if !reflect.DeepEqual(len(allPlugins), len(result)) {
+		t.Errorf("list plugins count mismatch: expected %v, got %v", allPlugins, result)
+		return
+	}
+outer:
+	for _, res := range result {
+		for _, p := range allPlugins {
+			if reflect.DeepEqual(p, res) {
+				continue outer
+			}
+		}
+		t.Errorf("list plugins mismatch: expected %v, got %v", allPlugins, result)
+		return
+	}
+
+	if !reflect.DeepEqual(expectedSources, r.sources) {
+		t.Errorf("sources mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedSources, r.sources)
+		return
+	}
+	if !reflect.DeepEqual(expectedFunctions, r.functions) {
+		t.Errorf("functions mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedFunctions, r.functions)
+		return
+	}
+	if !reflect.DeepEqual(expectedSinks, r.sinks) {
+		t.Errorf("sinks mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", expectedSinks, r.functions)
+		return
+	}
+	pn, ok := r.GetSymbol(plugin.SOURCE, "new")
+	if !ok {
+		t.Error("can't find symbol new")
+		return
+	}
+	if pn != "dummy" {
+		t.Errorf("GetSymbol wrong, expect dummy but got %s", pn)
+	}
+
+	// Delete concurrently
+	for n := range expectedPlugins {
+		wg.Add(1)
+		go func(name string) {
+			defer wg.Done()
+			r.Delete(name)
+		}(n)
+	}
+	wg.Wait()
+	result = r.List()
+	if !reflect.DeepEqual(0, len(result)) {
+		t.Errorf("list plugins count mismatch: expected no plugins, got %v", result)
+		return
+	}
+}

internal/plugin/portable/runtime/connection.go

@@ -0,0 +1,206 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"go.nanomsg.org/mangos/v3"
+	"go.nanomsg.org/mangos/v3/protocol/pull"
+	"go.nanomsg.org/mangos/v3/protocol/push"
+	"go.nanomsg.org/mangos/v3/protocol/rep"
+	_ "go.nanomsg.org/mangos/v3/transport/ipc"
+	"sync"
+	"time"
+)
+
+// Options Initialized in config
+var Options = map[string]interface{}{
+	mangos.OptionSendDeadline: 1000,
+}
+
+type Closable interface {
+	Close() error
+}
+
+type ControlChannel interface {
+	Handshake() error
+	SendCmd(arg []byte) error
+	Closable
+}
+
+// NanomsgReqChannel shared by symbols
+type NanomsgReqChannel struct {
+	sync.Mutex
+	sock mangos.Socket
+}
+
+func (r *NanomsgReqChannel) Close() error {
+	return r.sock.Close()
+}
+
+func (r *NanomsgReqChannel) SendCmd(arg []byte) error {
+	r.Lock()
+	defer r.Unlock()
+	if err := r.sock.Send(arg); err != nil {
+		return fmt.Errorf("can't send message on control rep socket: %s", err.Error())
+	}
+	if msg, err := r.sock.Recv(); err != nil {
+		return fmt.Errorf("can't receive: %s", err.Error())
+	} else {
+		if string(msg) != "ok" {
+			return fmt.Errorf("receive error: %s", string(msg))
+		}
+	}
+	return nil
+}
+
+// Handshake should only be called once
+func (r *NanomsgReqChannel) Handshake() error {
+	_, err := r.sock.Recv()
+	return err
+}
+
+type DataInChannel interface {
+	Recv() ([]byte, error)
+	Closable
+}
+
+type DataOutChannel interface {
+	Send([]byte) error
+	Closable
+}
+
+type DataReqChannel interface {
+	Handshake() error
+	Req([]byte) ([]byte, error)
+	Closable
+}
+
+type NanomsgReqRepChannel struct {
+	sync.Mutex
+	sock mangos.Socket
+}
+
+func (r *NanomsgReqRepChannel) Close() error {
+	return r.sock.Close()
+}
+
+func (r *NanomsgReqRepChannel) Req(arg []byte) ([]byte, error) {
+	r.Lock()
+	defer r.Unlock()
+	if err := r.sock.Send(arg); err != nil {
+		return nil, fmt.Errorf("can't send message on function rep socket: %s", err.Error())
+	}
+	return r.sock.Recv()
+}
+
+// Handshake should only be called once
+func (r *NanomsgReqRepChannel) Handshake() error {
+	_, err := r.sock.Recv()
+	return err
+}
+
+func CreateSourceChannel(ctx api.StreamContext) (DataInChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = pull.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new pull socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = listenWithRetry(sock, url); err != nil {
+		return nil, fmt.Errorf("can't listen on pull socket for %s: %s", url, err.Error())
+	}
+	return sock, nil
+}
+
+func CreateFunctionChannel(symbolName string) (DataReqChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = rep.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new rep socket: %s", err)
+	}
+	setSockOptions(sock)
+	sock.SetOption(mangos.OptionRecvDeadline, 1000)
+	url := fmt.Sprintf("ipc:///tmp/func_%s.ipc", symbolName)
+	if err = listenWithRetry(sock, url); err != nil {
+		return nil, fmt.Errorf("can't listen on rep socket for %s: %s", url, err.Error())
+	}
+	return &NanomsgReqRepChannel{sock: sock}, nil
+}
+
+func CreateSinkChannel(ctx api.StreamContext) (DataOutChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = push.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new push socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on push socket: %s", err.Error())
+	}
+	return sock, nil
+}
+
+func CreateControlChannel(pluginName string) (ControlChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = rep.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new rep socket: %s", err)
+	}
+	setSockOptions(sock)
+	sock.SetOption(mangos.OptionRecvDeadline, 100)
+	url := fmt.Sprintf("ipc:///tmp/plugin_%s.ipc", pluginName)
+	if err = listenWithRetry(sock, url); err != nil {
+		return nil, fmt.Errorf("can't listen on rep socket: %s", err.Error())
+	}
+	return &NanomsgReqChannel{sock: sock}, nil
+}
+
+func setSockOptions(sock mangos.Socket) {
+	for k, v := range Options {
+		sock.SetOption(k, v)
+	}
+}
+
+func listenWithRetry(sock mangos.Socket, url string) error {
+	var (
+		retryCount    = 300
+		retryInterval = 100
+	)
+	for {
+		err := sock.Listen(url)
+		if err == nil {
+			conf.Log.Infof("start to listen after %d tries", 301-retryCount)
+			return err
+		}
+		retryCount--
+		if retryCount < 0 {
+			return err
+		}
+		time.Sleep(time.Duration(retryInterval) * time.Millisecond)
+	}
+}

internal/plugin/portable/runtime/connection_test.go

@@ -0,0 +1,310 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/topo/context"
+	"github.com/lf-edge/ekuiper/internal/topo/state"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"go.nanomsg.org/mangos/v3"
+	"go.nanomsg.org/mangos/v3/protocol/pull"
+	"go.nanomsg.org/mangos/v3/protocol/push"
+	"go.nanomsg.org/mangos/v3/protocol/req"
+	"reflect"
+	"testing"
+)
+
+var okMsg = []byte("ok")
+
+func TestControlCh(t *testing.T) {
+	pluginName := "test"
+	// 1. normal process
+	ch, err := CreateControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("normal process: create channel error %v", err)
+		return
+	}
+	client, err := createMockControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("normal process: create client error %v", err)
+		return
+	}
+	clientStopped := false
+	go func() {
+		err := client.Run()
+		if err != nil && !clientStopped {
+			t.Errorf("normal process: client error %v", err)
+		}
+		fmt.Printf("exiting normal client\n")
+	}()
+
+	err = ch.Handshake()
+	if err != nil {
+		t.Errorf("normal process: handshake error %v", err)
+	}
+	sendCount := 0
+	for {
+		sendCount++
+		err = ch.SendCmd(okMsg)
+		if err != nil {
+			t.Errorf("normal process: %d sendCmd error %v", sendCount, err)
+		}
+		if sendCount >= 3 {
+			break
+		}
+	}
+	err = ch.Close()
+	if err != nil {
+		t.Errorf("normal process: close error %v", err)
+	}
+	// 2. client not closed, channel is still occupied?
+	ch, err = CreateControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("2nd process: recreate channel error %v", err)
+	}
+	// 3. server not started
+	err = ch.Close()
+	if err != nil {
+		t.Errorf("normal process: close error %v", err)
+	}
+	clientStopped = true
+	err = client.Close()
+	if err != nil {
+		t.Errorf("3rd process: close client error %v", err)
+	}
+	_, err = createMockControlChannel(pluginName)
+	if err == nil || err.Error() == "" {
+		t.Errorf("3rd process: create client should have error but got %v", err)
+		return
+	}
+	// 4. double control channel client
+	ch, err = CreateControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("4th process: create channel error %v", err)
+	}
+	client, err = createMockControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("4th process: create client error %v", err)
+	}
+	clientStopped = false
+	go func() {
+		err := client.Run()
+		if err != nil && !clientStopped {
+			t.Errorf("4th process: client error %v", err)
+		}
+		fmt.Printf("exiting 4th process client\n")
+	}()
+
+	// 5. no handshake
+	err = ch.SendCmd(okMsg)
+	if err == nil || err.Error() != "can't send message on control rep socket: incorrect protocol state" {
+		t.Errorf("5th process: send command should have error but got %v", err)
+	}
+	err = ch.Handshake()
+	if err != nil {
+		t.Errorf("5th process: handshake error %v", err)
+	}
+	err = ch.SendCmd(okMsg)
+	if err != nil {
+		t.Errorf("5th process: sendCmd error %v", err)
+	}
+	err = ch.Close()
+	if err != nil {
+		t.Errorf("5th process: close error %v", err)
+	}
+	clientStopped = true
+	err = client.Close()
+	if err != nil {
+		t.Errorf("5th process: client close error %v", err)
+	}
+}
+
+func TestDataIn(t *testing.T) {
+	i := 0
+	ctx := context.DefaultContext{}
+	sctx := ctx.WithMeta("rule1", "op1", &state.MemoryStore{}).WithInstance(1)
+	for i < 2 { // normal start and restart
+		ch, err := CreateSourceChannel(sctx)
+		if err != nil {
+			t.Errorf("phase %d create channel error %v", i, err)
+		}
+		client, err := createMockSourceChannel(sctx)
+		if err != nil {
+			t.Errorf("phase %d create client error %v", i, err)
+		}
+		go func() {
+			var c = 0
+			for c < 3 {
+				err := client.Send(okMsg)
+				if err != nil {
+					t.Errorf("phase %d client send error %v", i, err)
+					return
+				}
+				conf.Log.Debugf("phase %d sent %d messages", i, c)
+				c++
+			}
+		}()
+		var c = 0
+		for c < 3 {
+			msg, err := ch.Recv()
+			if err != nil {
+				t.Errorf("phase %d receive error %v", i, err)
+				return
+			}
+			if !reflect.DeepEqual(msg, okMsg) {
+				t.Errorf("phase %d receive %s but expect %s", i, msg, okMsg)
+			}
+			c++
+		}
+		err = ch.Close()
+		if err != nil {
+			t.Errorf("phase %d close error %v", i, err)
+		}
+		client.Close()
+		if err != nil {
+			t.Errorf("phase %d close client error %v", i, err)
+		}
+		i++
+	}
+}
+
+func TestDataOut(t *testing.T) {
+	i := 0
+	ctx := context.DefaultContext{}
+	sctx := ctx.WithMeta("rule1", "op1", &state.MemoryStore{}).WithInstance(1)
+	for i < 2 { // normal start and restart
+		client, err := createMockSinkChannel(sctx)
+		if err != nil {
+			t.Errorf("phase %d create client error %v", i, err)
+		}
+		ch, err := CreateSinkChannel(sctx)
+		if err != nil {
+			t.Errorf("phase %d create channel error %v", i, err)
+		}
+		go func() {
+			var c = 0
+			for c < 3 {
+				err := ch.Send(okMsg)
+				if err != nil {
+					t.Errorf("phase %d client send error %v", i, err)
+					return
+				}
+				conf.Log.Debugf("phase %d sent %d messages", i, c)
+				c++
+			}
+		}()
+		var c = 0
+		for c < 3 {
+			msg, err := client.Recv()
+			if err != nil {
+				t.Errorf("phase %d receive error %v", i, err)
+				return
+			}
+			if !reflect.DeepEqual(msg, okMsg) {
+				t.Errorf("phase %d receive %s but expect %s", i, msg, okMsg)
+			}
+			c++
+		}
+		err = ch.Close()
+		if err != nil {
+			t.Errorf("phase %d close error %v", i, err)
+		}
+		client.Close()
+		if err != nil {
+			t.Errorf("phase %d close client error %v", i, err)
+		}
+		i++
+	}
+}
+
+type mockControlClient struct {
+	sock mangos.Socket
+}
+
+// Run until process end
+func (r *mockControlClient) Run() error {
+	err := r.sock.Send([]byte("handshake"))
+	if err != nil {
+		return fmt.Errorf("can't send handshake: %s", err.Error())
+	}
+	for {
+		msg, err := r.sock.Recv()
+		if err != nil {
+			return fmt.Errorf("cannot receive on rep socket: %s", err.Error())
+		}
+		if !reflect.DeepEqual(msg, okMsg) {
+			return fmt.Errorf("control client recieve %s but expect %s", string(msg), string(okMsg))
+		}
+		err = r.sock.Send(okMsg)
+		if err != nil {
+			return fmt.Errorf("can't send reply: %s", err.Error())
+		}
+	}
+	return nil
+}
+
+func (r *mockControlClient) Close() error {
+	return r.sock.Close()
+}
+
+func createMockControlChannel(pluginName string) (*mockControlClient, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = req.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new req socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/plugin_%s.ipc", pluginName)
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
+	}
+	return &mockControlClient{sock: sock}, nil
+}
+
+func createMockSourceChannel(ctx api.StreamContext) (mangos.Socket, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = push.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new push socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on push socket: %s", err.Error())
+	}
+	return sock, nil
+}
+
+func createMockSinkChannel(ctx api.StreamContext) (mangos.Socket, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = pull.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new pull socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = listenWithRetry(sock, url); err != nil {
+		return nil, fmt.Errorf("can't listen on pull socket for %s: %s", url, err.Error())
+	}
+	return sock, nil
+}

internal/plugin/portable/runtime/function.go

@@ -0,0 +1,174 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	kctx "github.com/lf-edge/ekuiper/internal/topo/context"
+	"github.com/lf-edge/ekuiper/pkg/api"
+)
+
+// PortableFunc each function symbol only has a singleton
+// Each singleton are long running go routine.
+// TODO think about ending a portable func when needed.
+type PortableFunc struct {
+	symbolName string
+	reg        *PluginMeta
+	dataCh     DataReqChannel
+}
+
+func NewPortableFunc(symbolName string, reg *PluginMeta) (*PortableFunc, error) {
+	// Setup channel and route the data
+	conf.Log.Infof("Start running portable function meta %+v", reg)
+	pm := GetPluginInsManager()
+	ins, err := pm.getOrStartProcess(reg, PortbleConf)
+	if err != nil {
+		return nil, err
+	}
+	conf.Log.Infof("Plugin started successfully")
+
+	// Create function channel
+	dataCh, err := CreateFunctionChannel(symbolName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Start symbol
+	c := &Control{
+		SymbolName: symbolName,
+		PluginType: TYPE_FUNC,
+	}
+	ctx := kctx.WithValue(kctx.Background(), kctx.LoggerKey, conf.Log)
+	err = ins.StartSymbol(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	err = dataCh.Handshake()
+	if err != nil {
+		return nil, fmt.Errorf("function %s handshake error: %v", reg.Name, err)
+	}
+
+	return &PortableFunc{
+		symbolName: reg.Name,
+		reg:        reg,
+		dataCh:     dataCh,
+	}, nil
+}
+
+func (f *PortableFunc) Validate(args []interface{}) error {
+	// TODO function arg encoding
+	jsonArg, err := encode("Validate", args)
+	if err != nil {
+		return err
+	}
+	res, err := f.dataCh.Req(jsonArg)
+	if err != nil {
+		return err
+	}
+	fr := &FuncReply{}
+	err = json.Unmarshal(res, fr)
+	if err != nil {
+		return err
+	}
+	if fr.State {
+		return nil
+	} else {
+		return fmt.Errorf("validate return state is false, got %+v", fr)
+	}
+}
+
+func (f *PortableFunc) Exec(args []interface{}, ctx api.FunctionContext) (interface{}, bool) {
+	ctx.GetLogger().Debugf("running portable func with args %+v", args)
+	ctxRaw, err := encodeCtx(ctx)
+	if err != nil {
+		return err, false
+	}
+	jsonArg, err := encode("Exec", append(args, ctxRaw))
+	if err != nil {
+		return err, false
+	}
+	res, err := f.dataCh.Req(jsonArg)
+	if err != nil {
+		return err, false
+	}
+	fr := &FuncReply{}
+	err = json.Unmarshal(res, fr)
+	if err != nil {
+		return err, false
+	}
+	return fr.Result, fr.State
+}
+
+func (f *PortableFunc) IsAggregate() bool {
+	// TODO error handling
+	jsonArg, err := encode("IsAggregate", nil)
+	if err != nil {
+		conf.Log.Error(err)
+		return false
+	}
+	res, err := f.dataCh.Req(jsonArg)
+	if err != nil {
+		conf.Log.Error(err)
+		return false
+	}
+	fr := &FuncReply{}
+	err = json.Unmarshal(res, fr)
+	if err != nil {
+		conf.Log.Error(err)
+		return false
+	}
+	if fr.State {
+		r, ok := fr.Result.(bool)
+		if !ok {
+			conf.Log.Errorf("IsAggregate result is not bool, got %v", res)
+			return false
+		} else {
+			return r
+		}
+	} else {
+		conf.Log.Errorf("IsAggregate return state is false, got %+v", fr)
+		return false
+	}
+}
+
+func (f *PortableFunc) Close() error {
+	return f.dataCh.Close()
+	// Symbol must be cloased by instance manager
+	//		ins.StopSymbol(ctx, c)
+}
+
+func encode(funcName string, arg interface{}) ([]byte, error) {
+	c := FuncData{
+		Func: funcName,
+		Arg:  arg,
+	}
+	return json.Marshal(c)
+}
+
+func encodeCtx(ctx api.FunctionContext) (string, error) {
+	m := FuncMeta{
+		Meta: Meta{
+			RuleId:     ctx.GetRuleId(),
+			OpId:       ctx.GetOpId(),
+			InstanceId: ctx.GetInstanceId(),
+		},
+		FuncId: ctx.GetFuncId(),
+	}
+	bs, err := json.Marshal(m)
+	return string(bs), err
+}

internal/plugin/portable/runtime/plugin_ins_manager.go

@@ -0,0 +1,222 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"os"
+	"os/exec"
+	"sync"
+)
+
+var (
+	once sync.Once
+	pm   *pluginInsManager
+)
+
+// TODO setting configuration
+var PortbleConf = &PortableConfig{
+	SendTimeout: 1000,
+}
+
+type pluginIns struct {
+	process      *os.Process
+	ctrlChan     ControlChannel
+	runningCount int
+	name         string
+}
+
+func (i *pluginIns) StartSymbol(ctx api.StreamContext, ctrl *Control) error {
+	arg, err := json.Marshal(ctrl)
+	if err != nil {
+		return err
+	}
+	c := Command{
+		Cmd: CMD_START,
+		Arg: string(arg),
+	}
+	jsonArg, err := json.Marshal(c)
+	if err != nil {
+		return err
+	}
+	err = i.ctrlChan.SendCmd(jsonArg)
+	if err == nil {
+		i.runningCount++
+		ctx.GetLogger().Infof("started symbol %s", ctrl.SymbolName)
+	}
+	return err
+}
+
+func (i *pluginIns) StopSymbol(ctx api.StreamContext, ctrl *Control) error {
+	arg, err := json.Marshal(ctrl)
+	if err != nil {
+		return err
+	}
+	c := Command{
+		Cmd: CMD_STOP,
+		Arg: string(arg),
+	}
+	jsonArg, err := json.Marshal(c)
+	if err != nil {
+		return err
+	}
+	err = i.ctrlChan.SendCmd(jsonArg)
+	i.runningCount--
+	ctx.GetLogger().Infof("stopped symbol %s", ctrl.SymbolName)
+	if i.runningCount == 0 {
+		err := GetPluginInsManager().Kill(i.name)
+		if err != nil {
+			ctx.GetLogger().Infof("fail to stop plugin %s: %v", i.name, err)
+			return err
+		}
+		ctx.GetLogger().Infof("stop plugin %s", i.name)
+	}
+	return err
+}
+
+func (i *pluginIns) Stop() error {
+	_ = i.ctrlChan.Close()
+	err := i.process.Kill()
+	return err
+}
+
+// Manager plugin process and control socket
+type pluginInsManager struct {
+	instances map[string]*pluginIns
+	sync.RWMutex
+}
+
+func GetPluginInsManager() *pluginInsManager {
+	once.Do(func() {
+		pm = &pluginInsManager{
+			instances: make(map[string]*pluginIns),
+		}
+	})
+	return pm
+}
+
+func (p *pluginInsManager) getPluginIns(name string) (*pluginIns, bool) {
+	p.RLock()
+	defer p.RUnlock()
+	ins, ok := p.instances[name]
+	return ins, ok
+}
+
+func (p *pluginInsManager) deletePluginIns(name string) {
+	p.Lock()
+	defer p.Unlock()
+	delete(p.instances, name)
+}
+
+func (p *pluginInsManager) getOrStartProcess(pluginMeta *PluginMeta, pconf *PortableConfig) (*pluginIns, error) {
+	p.Lock()
+	defer p.Unlock()
+	if ins, ok := p.instances[pluginMeta.Name]; ok {
+		return ins, nil
+	}
+
+	conf.Log.Infof("create control channel")
+	ctrlChan, err := CreateControlChannel(pluginMeta.Name)
+	if err != nil {
+		return nil, fmt.Errorf("can't create new control channel: %s", err.Error())
+	}
+
+	conf.Log.Infof("executing plugin")
+	jsonArg, err := json.Marshal(pconf)
+	if err != nil {
+		return nil, fmt.Errorf("invalid conf: %v", pconf)
+	}
+	var cmd *exec.Cmd
+	switch pluginMeta.Language {
+	case "go":
+		conf.Log.Printf("starting go plugin executable %s", pluginMeta.Executable)
+		cmd = exec.Command(pluginMeta.Executable, string(jsonArg))
+
+	case "python":
+		conf.Log.Printf("starting python plugin executable %s\n", pluginMeta.Executable)
+		cmd = exec.Command("python", pluginMeta.Executable, string(jsonArg))
+	default:
+		return nil, fmt.Errorf("unsupported language: %s", pluginMeta.Language)
+	}
+	cmd.Stdout = conf.Log.Out
+	cmd.Stderr = conf.Log.Out
+
+	conf.Log.Println("plugin starting")
+	err = cmd.Start()
+	if err != nil {
+		return nil, fmt.Errorf("plugin executable %s stops with error %v", pluginMeta.Executable, err)
+	}
+	process := cmd.Process
+	conf.Log.Printf("plugin started pid: %d\n", process.Pid)
+	go func() {
+		err = cmd.Wait()
+		if err != nil {
+			conf.Log.Printf("plugin executable %s stops with error %v", pluginMeta.Executable, err)
+		}
+
+		if ins, ok := p.getPluginIns(pluginMeta.Name); ok {
+			_ = ins.ctrlChan.Close()
+			p.deletePluginIns(pluginMeta.Name)
+		}
+	}()
+
+	conf.Log.Println("waiting handshake")
+	err = ctrlChan.Handshake()
+	if err != nil {
+		return nil, fmt.Errorf("plugin %s control handshake error: %v", pluginMeta.Executable, err)
+	}
+
+	ins := &pluginIns{
+		name:     pluginMeta.Name,
+		process:  process,
+		ctrlChan: ctrlChan,
+	}
+	p.instances[pluginMeta.Name] = ins
+	conf.Log.Println("plugin start running")
+	return ins, nil
+}
+
+func (p *pluginInsManager) Kill(name string) error {
+	p.Lock()
+	defer p.Unlock()
+	var err error
+	if ins, ok := p.instances[name]; ok {
+		err = ins.Stop()
+		delete(p.instances, name)
+	} else {
+		return fmt.Errorf("instance %s not found", name)
+	}
+	return err
+}
+
+func (p *pluginInsManager) KillAll() error {
+	p.Lock()
+	defer p.Unlock()
+	for _, ins := range p.instances {
+		_ = ins.Stop()
+	}
+	p.instances = make(map[string]*pluginIns)
+	return nil
+}
+
+type PluginMeta struct {
+	Name       string `json:"name"`
+	Version    string `json:"version"`
+	Language   string `json:"language"`
+	Executable string `json:"executable"`
+}

internal/plugin/portable/runtime/plugin_ins_manager_test.go

@@ -0,0 +1,181 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/conf"
+	"github.com/lf-edge/ekuiper/internal/topo/context"
+	"github.com/lf-edge/ekuiper/internal/topo/state"
+	"go.nanomsg.org/mangos/v3"
+	"go.nanomsg.org/mangos/v3/protocol/req"
+	"testing"
+)
+
+// Plugin manager involves process, only covered in the integration test
+
+// TestPluginInstance test the encode/decode of command
+func TestPluginInstance(t *testing.T) {
+	pluginName := "test"
+	ch, err := CreateControlChannel(pluginName)
+	if err != nil {
+		t.Errorf("create channel error %v", err)
+		return
+	}
+	client, err := createMockClient(pluginName)
+	if err != nil {
+		t.Errorf("normal process: create client error %v", err)
+		return
+	}
+	err = client.Send([]byte("handshake"))
+	if err != nil {
+		t.Errorf("can't send handshake: %s", err.Error())
+		return
+	}
+	err = ch.Handshake()
+	if err != nil {
+		t.Errorf("can't ack handshake: %s", err.Error())
+		return
+	}
+	ins := &pluginIns{
+		name:     "test",
+		process:  nil,
+		ctrlChan: ch,
+	}
+	var tests = []struct {
+		c  *Control
+		sj string
+		ej string
+	}{
+		{
+			c: &Control{
+				SymbolName: "symbol1",
+				Meta: &Meta{
+					RuleId:     "rule1",
+					OpId:       "op1",
+					InstanceId: 0,
+				},
+				PluginType: "sources",
+				DataSource: "topic",
+				Config:     map[string]interface{}{"abc": 1},
+			},
+			sj: "{\"cmd\":\"start\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol1\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op1\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"sources\\\",\\\"dataSource\\\":\\\"topic\\\",\\\"config\\\":{\\\"abc\\\":1}}\"}",
+			ej: "{\"cmd\":\"stop\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol1\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op1\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"sources\\\",\\\"dataSource\\\":\\\"topic\\\",\\\"config\\\":{\\\"abc\\\":1}}\"}",
+		}, {
+			c: &Control{
+				SymbolName: "symbol2",
+				Meta: &Meta{
+					RuleId:     "rule1",
+					OpId:       "op2",
+					InstanceId: 0,
+				},
+				PluginType: "functions",
+			},
+			sj: "{\"cmd\":\"start\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol2\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op2\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"functions\\\"}\"}",
+			ej: "{\"cmd\":\"stop\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol2\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op2\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"functions\\\"}\"}",
+		}, {
+			c: &Control{
+				SymbolName: "symbol3",
+				Meta: &Meta{
+					RuleId:     "rule1",
+					OpId:       "op3",
+					InstanceId: 0,
+				},
+				PluginType: "sinks",
+				Config:     map[string]interface{}{"def": map[string]interface{}{"ci": "aaa"}},
+			},
+			sj: "{\"cmd\":\"start\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol3\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op3\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"sinks\\\",\\\"config\\\":{\\\"def\\\":{\\\"ci\\\":\\\"aaa\\\"}}}\"}",
+			ej: "{\"cmd\":\"stop\",\"arg\":\"{\\\"symbolName\\\":\\\"symbol3\\\",\\\"meta\\\":{\\\"ruleId\\\":\\\"rule1\\\",\\\"opId\\\":\\\"op3\\\",\\\"instanceId\\\":0},\\\"pluginType\\\":\\\"sinks\\\",\\\"config\\\":{\\\"def\\\":{\\\"ci\\\":\\\"aaa\\\"}}}\"}",
+		},
+	}
+	ctx := context.WithValue(context.Background(), context.LoggerKey, conf.Log)
+	sctx := ctx.WithMeta("rule1", "op1", &state.MemoryStore{}).WithInstance(1)
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	go func() {
+		err := ins.StartSymbol(sctx, tests[0].c)
+		if err != nil {
+			t.Errorf("start command err %v", err)
+			return
+		}
+		for _, tt := range tests {
+			err := ins.StartSymbol(sctx, tt.c)
+			if err != nil {
+				t.Errorf("start command err %v", err)
+				return
+			}
+			err = ins.StopSymbol(sctx, tt.c)
+			if err != nil {
+				t.Errorf("stop command err %v", err)
+				return
+			}
+		}
+	}()
+	// start symbol1 to avoild instance clean
+	msg, err := client.Recv()
+	if err != nil {
+		t.Errorf("receive start command err %v", err)
+	}
+	client.Send(okMsg)
+	sj := string(msg)
+	if sj != tests[0].sj {
+		t.Errorf("start command mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", tests[0].sj, sj)
+	}
+	for _, tt := range tests {
+		msg, err := client.Recv()
+		if err != nil {
+			t.Errorf("receive start command err %v", err)
+			break
+		}
+		client.Send(okMsg)
+		sj := string(msg)
+		if sj != tt.sj {
+			t.Errorf("start command mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", tt.sj, sj)
+		}
+		msg, err = client.Recv()
+		if err != nil {
+			t.Errorf("receive stop command err %v", err)
+			break
+		}
+		client.Send(okMsg)
+		ej := string(msg)
+		if ej != tt.ej {
+			t.Errorf("end command mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", tt.ej, ej)
+		}
+	}
+	err = client.Close()
+	if err != nil {
+		t.Errorf("close client error %v", err)
+	}
+	err = ins.ctrlChan.Close()
+	if err != nil {
+		t.Errorf("close ins error %v", err)
+	}
+}
+
+func createMockClient(pluginName string) (mangos.Socket, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = req.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new req socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/plugin_%s.ipc", pluginName)
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
+	}
+	return sock, nil
+}

internal/plugin/portable/runtime/shared.go

@@ -0,0 +1,68 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+const (
+	TYPE_SOURCE = "source"
+	TYPE_SINK   = "sink"
+	TYPE_FUNC   = "func"
+)
+
+type Meta struct {
+	RuleId     string `json:"ruleId"`
+	OpId       string `json:"opId"`
+	InstanceId int    `json:"instanceId"`
+}
+
+type FuncMeta struct {
+	Meta
+	FuncId int `json:"funcId"`
+}
+
+type Control struct {
+	SymbolName string                 `json:"symbolName"`
+	Meta       *Meta                  `json:"meta,omitempty"`
+	PluginType string                 `json:"pluginType"`
+	DataSource string                 `json:"dataSource,omitempty"`
+	Config     map[string]interface{} `json:"config,omitempty"`
+}
+
+type Command struct {
+	Cmd string `json:"cmd"`
+	Arg string `json:"arg"`
+}
+
+const (
+	CMD_START = "start"
+	CMD_STOP  = "stop"
+)
+
+const (
+	REPLY_OK = "ok"
+)
+
+type PortableConfig struct {
+	SendTimeout int64 `json:"sendTimeout"`
+}
+
+type FuncData struct {
+	Func string      `json:"func"`
+	Arg  interface{} `json:"arg"`
+}
+
+type FuncReply struct {
+	State  bool        `json:"state"`
+	Result interface{} `json:"result"`
+}

internal/plugin/portable/runtime/sink.go

@@ -0,0 +1,95 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/pkg/api"
+)
+
+type PortableSink struct {
+	symbolName string
+	reg        *PluginMeta
+	props      map[string]interface{}
+	dataCh     DataOutChannel
+	clean      func() error
+}
+
+func NewPortableSink(symbolName string, reg *PluginMeta) *PortableSink {
+	return &PortableSink{
+		symbolName: symbolName,
+		reg:        reg,
+	}
+}
+
+func (ps *PortableSink) Configure(props map[string]interface{}) error {
+	ps.props = props
+	return nil
+}
+
+func (ps *PortableSink) Open(ctx api.StreamContext) error {
+	ctx.GetLogger().Infof("Start running portable sink %s with conf %+v", ps.symbolName, ps.props)
+	pm := GetPluginInsManager()
+	ins, err := pm.getOrStartProcess(ps.reg, PortbleConf)
+	if err != nil {
+		return err
+	}
+	ctx.GetLogger().Infof("Plugin started successfully")
+
+	// Control: send message to plugin to ask starting symbol
+	c := &Control{
+		Meta: &Meta{
+			RuleId:     ctx.GetRuleId(),
+			OpId:       ctx.GetOpId(),
+			InstanceId: ctx.GetInstanceId(),
+		},
+		SymbolName: ps.symbolName,
+		PluginType: TYPE_SINK,
+		Config:     ps.props,
+	}
+	err = ins.StartSymbol(ctx, c)
+	if err != nil {
+		return err
+	}
+
+	// must start symbol firstly
+	dataCh, err := CreateSinkChannel(ctx)
+	if err != nil {
+		return err
+	}
+
+	ps.clean = func() error {
+		ctx.GetLogger().Info("closing sink data channe")
+		dataCh.Close()
+		return ins.StopSymbol(ctx, c)
+	}
+	ps.dataCh = dataCh
+	return nil
+}
+
+func (ps *PortableSink) Collect(ctx api.StreamContext, item interface{}) error {
+	ctx.GetLogger().Debugf("Receive %+v", item)
+	// TODO item type
+	switch input := item.(type) {
+	case []byte:
+		return ps.dataCh.Send(input)
+	default:
+		return ps.dataCh.Send([]byte(fmt.Sprintf("%v", input)))
+	}
+}
+
+func (ps *PortableSink) Close(ctx api.StreamContext) error {
+	return ps.clean()
+}

internal/plugin/portable/runtime/source.go

@@ -0,0 +1,112 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/pkg/api"
+)
+
+// Error handling: wrap all error in a function to handle
+
+type PortableSource struct {
+	symbolName string
+	reg        *PluginMeta
+
+	topic string
+	props map[string]interface{}
+}
+
+func NewPortableSource(symbolName string, reg *PluginMeta) *PortableSource {
+	return &PortableSource{
+		symbolName: symbolName,
+		reg:        reg,
+	}
+}
+
+func (ps *PortableSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple, errCh chan<- error) {
+	ctx.GetLogger().Infof("Start running portable source %s with datasource %s and conf %+v", ps.symbolName, ps.topic, ps.props)
+	pm := GetPluginInsManager()
+	ins, err := pm.getOrStartProcess(ps.reg, PortbleConf)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	ctx.GetLogger().Infof("Plugin started successfully")
+
+	// wait for plugin data
+	dataCh, err := CreateSourceChannel(ctx)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	defer func() {
+		ctx.GetLogger().Info("Closing source data channel")
+		dataCh.Close()
+	}()
+
+	// Control: send message to plugin to ask starting symbol
+	c := &Control{
+		Meta: &Meta{
+			RuleId:     ctx.GetRuleId(),
+			OpId:       ctx.GetOpId(),
+			InstanceId: ctx.GetInstanceId(),
+		},
+		SymbolName: ps.symbolName,
+		PluginType: TYPE_SOURCE,
+		DataSource: ps.topic,
+		Config:     ps.props,
+	}
+	err = ins.StartSymbol(ctx, c)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	defer ins.StopSymbol(ctx, c)
+
+	for {
+		var msg []byte
+		msg, err = dataCh.Recv()
+		if err != nil {
+			errCh <- fmt.Errorf("cannot receive from mangos Socket: %s", err.Error())
+			return
+		}
+		result := &api.DefaultSourceTuple{}
+		e := json.Unmarshal(msg, result)
+		if e != nil {
+			ctx.GetLogger().Errorf("Invalid data format, cannot decode %s to json format with error %s", string(msg), e)
+			continue
+		}
+		select {
+		case consumer <- result:
+			ctx.GetLogger().Debugf("send data to source node")
+		case <-ctx.Done():
+			ctx.GetLogger().Info("stop source")
+			return
+		}
+	}
+}
+
+func (ps *PortableSource) Configure(topic string, props map[string]interface{}) error {
+	ps.topic = topic
+	ps.props = props
+	return nil
+}
+
+func (ps *PortableSource) Close(ctx api.StreamContext) error {
+	ctx.GetLogger().Infof("Closing source %s", ps.symbolName)
+	return nil
+}

internal/plugin/portable/test/portable_rule_test.go

@@ -0,0 +1,173 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/internal/binder/io"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
+	"github.com/lf-edge/ekuiper/internal/processor"
+	"github.com/lf-edge/ekuiper/internal/topo/planner"
+	"github.com/lf-edge/ekuiper/internal/topo/topotest"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"log"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func init() {
+	m, err := portable.InitManager()
+	if err != nil {
+		panic(err)
+	}
+	entry := binder.FactoryEntry{Name: "portable plugin", Factory: m}
+	err = function.Initialize([]binder.FactoryEntry{entry})
+	if err != nil {
+		panic(err)
+	}
+	err = io.Initialize([]binder.FactoryEntry{entry})
+	if err != nil {
+		panic(err)
+	}
+}
+
+const CACHE_FILE = "cache2"
+
+func TestSourceAndFunc(t *testing.T) {
+	streamList := []string{"ext"}
+	topotest.HandleStream(false, streamList, t)
+	var tests = []struct {
+		Name string
+		Rule string
+		R    [][]map[string]interface{}
+		M    map[string]interface{}
+	}{
+		{
+			Name: `TestPortableRule1`,
+			Rule: `{"sql":"SELECT echo(count) as ee FROM ext","actions":[{"file":{"path":"` + CACHE_FILE + `"}}]}`,
+			R: [][]map[string]interface{}{
+				{{
+					"ee": float64(50),
+				}},
+				{{
+					"ee": float64(50),
+				}},
+				{{
+					"ee": float64(50),
+				}},
+			},
+			M: map[string]interface{}{
+				"source_ext_0_exceptions_total":   int64(0),
+				"source_ext_0_records_in_total":   int64(3),
+				"source_ext_0_records_out_total":  int64(3),
+				"sink_file_0_0_records_out_total": int64(3),
+			},
+		},
+	}
+	topotest.HandleStream(true, streamList, t)
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	defer runtime.GetPluginInsManager().KillAll()
+	for i, tt := range tests {
+		_ = os.Remove(CACHE_FILE)
+		rs, err := CreateRule(tt.Name, tt.Rule)
+		if err != nil {
+			t.Errorf("failed to create rule: %s.", err)
+			continue
+		}
+		tp, err := planner.Plan(rs)
+		if err != nil {
+			t.Errorf("fail to init rule: %v", err)
+			continue
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		go func(ctx context.Context) {
+			select {
+			case err := <-tp.Open():
+				log.Println(err)
+				tp.Cancel()
+			case <-ctx.Done():
+				log.Printf("ctx done %v\n", ctx.Err())
+				tp.Cancel()
+			}
+			fmt.Println("all exit")
+		}(ctx)
+		for {
+			if ctx.Err() != nil {
+				t.Errorf("Exiting with error %v", ctx.Err())
+				break
+			}
+			time.Sleep(10 * time.Millisecond)
+			if err := topotest.CompareMetrics(tp, tt.M); err == nil {
+				cancel()
+				// need to wait for file results
+				time.Sleep(10 * time.Millisecond)
+				results := getResults()
+				fmt.Printf("get results %v\n", results)
+				time.Sleep(10 * time.Millisecond)
+				var mm [][]map[string]interface{}
+				for i, v := range results {
+					if i >= 3 {
+						break
+					}
+					var mapRes []map[string]interface{}
+					err := json.Unmarshal([]byte(v), &mapRes)
+					if err != nil {
+						t.Errorf("Failed to parse the input into map")
+						continue
+					}
+					mm = append(mm, mapRes)
+				}
+				if !reflect.DeepEqual(tt.R, mm) {
+					t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.Rule, tt.R, results)
+				}
+				break
+			}
+		}
+	}
+	topotest.HandleStream(false, streamList, t)
+	// wait for rule clean up
+	time.Sleep(1 * time.Second)
+}
+
+func getResults() []string {
+	f, err := os.Open(CACHE_FILE)
+	if err != nil {
+		panic(err)
+	}
+	result := make([]string, 0)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		result = append(result, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		panic(err)
+	}
+	f.Close()
+	return result
+}
+
+func CreateRule(name, sql string) (*api.Rule, error) {
+	p := processor.NewRuleProcessor()
+	p.ExecDrop(name)
+	return p.ExecCreate(name, sql)
+}

internal/server/rest.go

@@ -22,7 +22,9 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/lf-edge/ekuiper/internal/conf"
 	"github.com/lf-edge/ekuiper/internal/meta"
+	"github.com/lf-edge/ekuiper/internal/plugin"
 	"github.com/lf-edge/ekuiper/internal/plugin/native"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/lf-edge/ekuiper/pkg/ast"
@@ -108,7 +110,6 @@ func createRestServer(ip string, port int) *http.Server {
 	r.HandleFunc("/plugins/sources", sourcesHandler).Methods(http.MethodGet, http.MethodPost)
 	r.HandleFunc("/plugins/sources/prebuild", prebuildSourcePlugins).Methods(http.MethodGet)
 	r.HandleFunc("/plugins/sources/{name}", sourceHandler).Methods(http.MethodDelete, http.MethodGet)
-
 	r.HandleFunc("/plugins/sinks", sinksHandler).Methods(http.MethodGet, http.MethodPost)
 	r.HandleFunc("/plugins/sinks/prebuild", prebuildSinkPlugins).Methods(http.MethodGet)
 	r.HandleFunc("/plugins/sinks/{name}", sinkHandler).Methods(http.MethodDelete, http.MethodGet)
@@ -119,11 +120,12 @@ func createRestServer(ip string, port int) *http.Server {
 	r.HandleFunc("/plugins/udfs", functionsListHandler).Methods(http.MethodGet)
 	r.HandleFunc("/plugins/udfs/{name}", functionsGetHandler).Methods(http.MethodGet)
 
-	r.HandleFunc("/metadata/functions", functionsMetaHandler).Methods(http.MethodGet)
+	r.HandleFunc("/plugins/portables", portablesHandler).Methods(http.MethodGet, http.MethodPost)
+	r.HandleFunc("/plugins/portables/{name}", portableHandler).Methods(http.MethodGet, http.MethodDelete)
 
+	r.HandleFunc("/metadata/functions", functionsMetaHandler).Methods(http.MethodGet)
 	r.HandleFunc("/metadata/sinks", sinksMetaHandler).Methods(http.MethodGet)
 	r.HandleFunc("/metadata/sinks/{name}", newSinkMetaHandler).Methods(http.MethodGet)
-
 	r.HandleFunc("/metadata/sources", sourcesMetaHandler).Methods(http.MethodGet)
 	r.HandleFunc("/metadata/sources/yaml/{name}", sourceConfHandler).Methods(http.MethodGet)
 	r.HandleFunc("/metadata/sources/{name}", sourceMetaHandler).Methods(http.MethodGet)
@@ -422,7 +424,7 @@ func getTopoRuleHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(content))
 }
 
-func pluginsHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
+func pluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	pluginManager := native.GetManager()
 	defer r.Body.Close()
 	switch r.Method {
@@ -430,24 +432,24 @@ func pluginsHandler(w http.ResponseWriter, r *http.Request, t native.PluginType)
 		content := pluginManager.List(t)
 		jsonResponse(content, w, logger)
 	case http.MethodPost:
-		sd := native.NewPluginByType(t)
+		sd := plugin.NewPluginByType(t)
 		err := json.NewDecoder(r.Body).Decode(sd)
 		// Problems decoding
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", native.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("Invalid body: Error decoding the %s plugin json", plugin.PluginTypes[t]), logger)
 			return
 		}
 		err = pluginManager.Register(t, sd)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("%s plugins create command error", native.PluginTypes[t]), logger)
+			handleError(w, err, fmt.Sprintf("%s plugins create command error", plugin.PluginTypes[t]), logger)
 			return
 		}
 		w.WriteHeader(http.StatusCreated)
-		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", native.PluginTypes[t], sd.GetName())))
+		w.Write([]byte(fmt.Sprintf("%s plugin %s is created", plugin.PluginTypes[t], sd.GetName())))
 	}
 }
 
-func pluginHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
+func pluginHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	defer r.Body.Close()
 	vars := mux.Vars(r)
 	name := vars["name"]
@@ -458,11 +460,11 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t native.PluginType)
 		r := cb == "1"
 		err := pluginManager.Delete(t, name, r)
 		if err != nil {
-			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", native.PluginTypes[t], name), logger)
+			handleError(w, err, fmt.Sprintf("delete %s plugin %s error", plugin.PluginTypes[t], name), logger)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
-		result := fmt.Sprintf("%s plugin %s is deleted", native.PluginTypes[t], name)
+		result := fmt.Sprintf("%s plugin %s is deleted", plugin.PluginTypes[t], name)
 		if r {
 			result = fmt.Sprintf("%s and Kuiper will be stopped", result)
 		} else {
@@ -472,7 +474,7 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t native.PluginType)
 	case http.MethodGet:
 		j, ok := pluginManager.GetPluginInfo(t, name)
 		if !ok {
-			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", native.PluginTypes[t], name), logger)
+			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe %s plugin %s error", plugin.PluginTypes[t], name), logger)
 			return
 		}
 		jsonResponse(j, w, logger)
@@ -481,27 +483,27 @@ func pluginHandler(w http.ResponseWriter, r *http.Request, t native.PluginType)
 
 //list or create source plugin
 func sourcesHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, native.SOURCE)
+	pluginsHandler(w, r, plugin.SOURCE)
 }
 
 //delete a source plugin
 func sourceHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, native.SOURCE)
+	pluginHandler(w, r, plugin.SOURCE)
 }
 
 //list or create sink plugin
 func sinksHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, native.SINK)
+	pluginsHandler(w, r, plugin.SINK)
 }
 
 //delete a sink plugin
 func sinkHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, native.SINK)
+	pluginHandler(w, r, plugin.SINK)
 }
 
 //list or create function plugin
 func functionsHandler(w http.ResponseWriter, r *http.Request) {
-	pluginsHandler(w, r, native.FUNCTION)
+	pluginsHandler(w, r, plugin.FUNCTION)
 }
 
 //list all user defined functions in all function plugins
@@ -515,7 +517,7 @@ func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	name := vars["name"]
 	pluginManager := native.GetManager()
-	j, ok := pluginManager.GetPluginBySymbol(native.FUNCTION, name)
+	j, ok := pluginManager.GetPluginBySymbol(plugin.FUNCTION, name)
 	if !ok {
 		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe function %s error", name), logger)
 		return
@@ -525,7 +527,7 @@ func functionsGetHandler(w http.ResponseWriter, r *http.Request) {
 
 //delete a function plugin
 func functionHandler(w http.ResponseWriter, r *http.Request) {
-	pluginHandler(w, r, native.FUNCTION)
+	pluginHandler(w, r, plugin.FUNCTION)
 }
 
 type functionList struct {
@@ -540,9 +542,9 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	name := vars["name"]
 	pluginManager := native.GetManager()
-	_, ok := pluginManager.GetPluginInfo(native.FUNCTION, name)
+	_, ok := pluginManager.GetPluginInfo(plugin.FUNCTION, name)
 	if !ok {
-		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", native.PluginTypes[native.FUNCTION], name), logger)
+		handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("register %s plugin %s error", plugin.PluginTypes[plugin.FUNCTION], name), logger)
 		return
 	}
 	sd := functionList{}
@@ -561,16 +563,66 @@ func functionRegisterHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(fmt.Sprintf("function plugin %s function list is registered", name)))
 }
 
+func portablesHandler(w http.ResponseWriter, r *http.Request) {
+	m := portable.GetManager()
+	defer r.Body.Close()
+	switch r.Method {
+	case http.MethodGet:
+		content := m.List()
+		jsonResponse(content, w, logger)
+	case http.MethodPost:
+		sd := plugin.NewPluginByType(plugin.PORTABLE)
+		err := json.NewDecoder(r.Body).Decode(sd)
+		// Problems decoding
+		if err != nil {
+			handleError(w, err, "Invalid body: Error decoding the portable plugin json", logger)
+			return
+		}
+		err = m.Register(sd)
+		if err != nil {
+			handleError(w, err, "portable plugin create command error", logger)
+			return
+		}
+		w.WriteHeader(http.StatusCreated)
+		w.Write([]byte(fmt.Sprintf("portable plugin %s is created", sd.GetName())))
+	}
+}
+
+func portableHandler(w http.ResponseWriter, r *http.Request) {
+	defer r.Body.Close()
+	vars := mux.Vars(r)
+	name := vars["name"]
+	m := portable.GetManager()
+	switch r.Method {
+	case http.MethodDelete:
+		err := m.Delete(name)
+		if err != nil {
+			handleError(w, err, fmt.Sprintf("delete portable plugin %s error", name), logger)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		result := fmt.Sprintf("portable plugin %s is deleted", name)
+		w.Write([]byte(result))
+	case http.MethodGet:
+		j, ok := m.GetPluginInfo(name)
+		if !ok {
+			handleError(w, errorx.NewWithCode(errorx.NOT_FOUND, "not found"), fmt.Sprintf("describe portable plugin %s error", name), logger)
+			return
+		}
+		jsonResponse(j, w, logger)
+	}
+}
+
 func prebuildSourcePlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, native.SOURCE)
+	prebuildPluginsHandler(w, r, plugin.SOURCE)
 }
 
 func prebuildSinkPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, native.SINK)
+	prebuildPluginsHandler(w, r, plugin.SINK)
 }
 
 func prebuildFuncsPlugins(w http.ResponseWriter, r *http.Request) {
-	prebuildPluginsHandler(w, r, native.FUNCTION)
+	prebuildPluginsHandler(w, r, plugin.FUNCTION)
 }
 
 func isOffcialDockerImage() bool {
@@ -580,7 +632,7 @@ func isOffcialDockerImage() bool {
 	return true
 }
 
-func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t native.PluginType) {
+func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t plugin.PluginType) {
 	emsg := "It's strongly recommended to install plugins at official released Debian Docker images. If you choose to proceed to install plugin, please make sure the plugin is already validated in your own build."
 	if !isOffcialDockerImage() {
 		handleError(w, fmt.Errorf(emsg), "", logger)
@@ -596,9 +648,9 @@ func prebuildPluginsHandler(w http.ResponseWriter, r *http.Request, t native.Plu
 		if strings.Contains(prettyName, "DEBIAN") {
 			hosts := conf.Config.Basic.PluginHosts
 			ptype := "sources"
-			if t == native.SINK {
+			if t == plugin.SINK {
 				ptype = "sinks"
-			} else if t == native.FUNCTION {
+			} else if t == plugin.FUNCTION {
 				ptype = "functions"
 			}
 			if err, plugins := fetchPluginList(hosts, ptype, os, runtime.GOARCH); err != nil {

internal/server/rpc.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/lf-edge/ekuiper/internal/pkg/model"
+	"github.com/lf-edge/ekuiper/internal/plugin"
 	"github.com/lf-edge/ekuiper/internal/plugin/native"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/lf-edge/ekuiper/internal/topo/sink"
@@ -202,7 +203,7 @@ func (t *Server) DropRule(name string, reply *string) error {
 }
 
 func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
-	pt := native.PluginType(arg.Type)
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
@@ -220,7 +221,7 @@ func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) RegisterPlugin(arg *model.PluginDesc, reply *string) error {
-	p, err := getPluginByJson(arg, native.FUNCTION)
+	p, err := getPluginByJson(arg, plugin.FUNCTION)
 	if err != nil {
 		return fmt.Errorf("Register plugin functions error: %s", err)
 	}
@@ -237,7 +238,7 @@ func (t *Server) RegisterPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) DropPlugin(arg *model.PluginDesc, reply *string) error {
-	pt := native.PluginType(arg.Type)
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Drop plugin error: %s", err)
@@ -257,7 +258,7 @@ func (t *Server) DropPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) ShowPlugins(arg int, reply *string) error {
-	pt := native.PluginType(arg)
+	pt := plugin.PluginType(arg)
 	l := native.GetManager().List(pt)
 	if len(l) == 0 {
 		l = append(l, "No plugin is found.")
@@ -276,7 +277,7 @@ func (t *Server) ShowUdfs(_ int, reply *string) error {
 }
 
 func (t *Server) DescPlugin(arg *model.PluginDesc, reply *string) error {
-	pt := native.PluginType(arg.Type)
+	pt := plugin.PluginType(arg.Type)
 	p, err := getPluginByJson(arg, pt)
 	if err != nil {
 		return fmt.Errorf("Describe plugin error: %s", err)
@@ -295,7 +296,7 @@ func (t *Server) DescPlugin(arg *model.PluginDesc, reply *string) error {
 }
 
 func (t *Server) DescUdf(arg string, reply *string) error {
-	m, ok := native.GetManager().GetPluginBySymbol(native.FUNCTION, arg)
+	m, ok := native.GetManager().GetPluginBySymbol(plugin.FUNCTION, arg)
 	if !ok {
 		return fmt.Errorf("Describe udf error: not found")
 	} else {
@@ -417,8 +418,8 @@ func marshalDesc(m interface{}) (string, error) {
 	return dst.String(), nil
 }
 
-func getPluginByJson(arg *model.PluginDesc, pt native.PluginType) (native.Plugin, error) {
-	p := native.NewPluginByType(pt)
+func getPluginByJson(arg *model.PluginDesc, pt plugin.PluginType) (plugin.Plugin, error) {
+	p := plugin.NewPluginByType(pt)
 	if arg.Json != "" {
 		if err := json.Unmarshal([]byte(arg.Json), p); err != nil {
 			return nil, fmt.Errorf("Parse plugin %s error : %s.", arg.Json, err)

internal/server/ruleManager.go

@@ -107,12 +107,12 @@ func doStartRule(rs *RuleState) error {
 		case err := <-tp.Open():
 			if err != nil {
 				tp.GetContext().SetError(err)
-				logger.Printf("closing rule %s for error: %v", rs.Name, err)
+				logger.Errorf("closing rule %s for error: %v", rs.Name, err)
 				tp.Cancel()
 				rs.Triggered = false
 			} else {
 				rs.Triggered = false
-				logger.Printf("closing rule %s", rs.Name)
+				logger.Infof("closing rule %s", rs.Name)
 			}
 		}
 	}()

internal/server/server.go

@@ -22,6 +22,8 @@ import (
 	"github.com/lf-edge/ekuiper/internal/conf"
 	"github.com/lf-edge/ekuiper/internal/pkg/store"
 	"github.com/lf-edge/ekuiper/internal/plugin/native"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable/runtime"
 	"github.com/lf-edge/ekuiper/internal/processor"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -63,12 +65,17 @@ func StartUp(Version, LoadFileType string) {
 	if err != nil {
 		panic(err)
 	}
+	portableManager, err := portable.InitManager()
+	if err != nil {
+		panic(err)
+	}
 	serviceManager, err := service.InitManager()
 	if err != nil {
 		panic(err)
 	}
 	entries := []binder.FactoryEntry{
 		{Name: "native plugin", Factory: nativeManager},
+		{Name: "portable plugin", Factory: portableManager},
 		{Name: "external service", Factory: serviceManager},
 	}
 	err = function.Initialize(entries)
@@ -174,6 +181,8 @@ func StartUp(Version, LoadFileType string) {
 	signal.Notify(sigint, os.Interrupt, syscall.SIGTERM)
 	<-sigint
 
+	runtime.GetPluginInsManager().KillAll()
+
 	if err = srvRpc.Shutdown(context.TODO()); err != nil {
 		logger.Errorf("rpc server shutdown error: %v", err)
 	}

internal/topo/topotest/mock_topo.go

@@ -49,7 +49,7 @@ type RuleTest struct {
 	W    int                    // wait time for each data sending, in milli
 }
 
-func compareMetrics(tp *topo.Topo, m map[string]interface{}) (err error) {
+func CompareMetrics(tp *topo.Topo, m map[string]interface{}) (err error) {
 	keys, values := tp.GetMetrics()
 	for k, v := range m {
 		var (
@@ -91,7 +91,7 @@ func compareMetrics(tp *topo.Topo, m map[string]interface{}) (err error) {
 	return nil
 }
 
-func commonResultFunc(result [][]byte) interface{} {
+func CommonResultFunc(result [][]byte) interface{} {
 	var maps [][]map[string]interface{}
 	for _, v := range result {
 		var mapRes []map[string]interface{}
@@ -105,7 +105,7 @@ func commonResultFunc(result [][]byte) interface{} {
 }
 
 func DoRuleTest(t *testing.T, tests []RuleTest, j int, opt *api.RuleOption, wait int) {
-	doRuleTestBySinkProps(t, tests, j, opt, wait, nil, commonResultFunc)
+	doRuleTestBySinkProps(t, tests, j, opt, wait, nil, CommonResultFunc)
 }
 
 func doRuleTestBySinkProps(t *testing.T, tests []RuleTest, j int, opt *api.RuleOption, w int, sinkProps map[string]interface{}, resultFunc func(result [][]byte) interface{}) {
@@ -161,7 +161,7 @@ func compareResult(t *testing.T, mockSink *mocknode.MockSink, resultFunc func(re
 	if !reflect.DeepEqual(tt.R, maps) {
 		t.Errorf("%d. %q\n\nresult mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.Sql, tt.R, maps)
 	}
-	if err := compareMetrics(tp, tt.M); err != nil {
+	if err := CompareMetrics(tp, tt.M); err != nil {
 		t.Errorf("%d. %q\n\nmetrics mismatch:\n\n%s\n\n", i, tt.Sql, err)
 	}
 	if tt.T != nil {
@@ -207,7 +207,7 @@ func sendData(t *testing.T, dataLength int, metrics map[string]interface{}, data
 	time.Sleep(10 * time.Millisecond)
 	var retry int
 	for retry = 4; retry > 0; retry-- {
-		if err := compareMetrics(tp, metrics); err == nil {
+		if err := CompareMetrics(tp, metrics); err == nil {
 			break
 		} else {
 			conf.Log.Errorf("check metrics error at %d: %s", retry, err)
@@ -439,7 +439,7 @@ func DoCheckpointRuleTest(t *testing.T, tests []RuleCheckpointTest, j int, opt *
 			t.Errorf("second phase send data error %s", err)
 			break
 		}
-		compareResult(t, mockSink, commonResultFunc, tt.RuleTest, i, tp)
+		compareResult(t, mockSink, CommonResultFunc, tt.RuleTest, i, tp)
 	}
 }
 

internal/xsql/func_invoker_test.go

@@ -0,0 +1,38 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package xsql
+
+import (
+	"github.com/lf-edge/ekuiper/internal/binder"
+	"github.com/lf-edge/ekuiper/internal/binder/function"
+	"github.com/lf-edge/ekuiper/pkg/ast"
+	"testing"
+)
+
+func TestInvoke(t *testing.T) {
+	err := function.Initialize([]binder.FactoryEntry{})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	err = validateFuncs("sum", []ast.Expr{})
+	if err == nil || err.Error() != "The arguments for sum should be 1." {
+		t.Error(err)
+	}
+	err = validateFuncs("window_start", []ast.Expr{})
+	if err != nil {
+		t.Error(err)
+	}
+}

pkg/api/stream.go

@@ -25,22 +25,22 @@ type SourceTuple interface {
 }
 
 type DefaultSourceTuple struct {
-	message map[string]interface{}
-	meta    map[string]interface{}
+	Mess map[string]interface{} `json:"message"`
+	M    map[string]interface{} `json:"meta"`
 }
 
 func NewDefaultSourceTuple(message map[string]interface{}, meta map[string]interface{}) *DefaultSourceTuple {
 	return &DefaultSourceTuple{
-		message: message,
-		meta:    meta,
+		Mess: message,
+		M:    meta,
 	}
 }
 
 func (t *DefaultSourceTuple) Message() map[string]interface{} {
-	return t.message
+	return t.Mess
 }
 func (t *DefaultSourceTuple) Meta() map[string]interface{} {
-	return t.meta
+	return t.M
 }
 
 type Logger interface {

plugins/portable/readme.md

@@ -0,0 +1,62 @@
+# Load Portable Plugin by File
+
+There are 2 ways to install portable plugins. One is to install by REST/CLI API. Another is to put all the plugin files with specified format into this path 'plugins/portable'.
+
+## Portable Plugin Composition
+
+There are two levels of a portable plugin. Each plugin has one single executable which will be executed as a separated process in runtime. Uses can define multiple `symbols` inside the one plugin. Each symbol could be a source, sink or function. Thus, when defining a portable plugin, users can get a set of new source, sink and function registered.
+
+For example, users can define a plugin named `car` and export many symbols for source, sink and function. The definition will be presented as a json file as below:
+
+```json
+{
+  "name": "car",
+  "version": "v1.0.0",
+  "language": "go",
+  "executable": "server",
+  "sources": [
+    "json","udp","sync"
+  ],
+  "sinks": [
+    "command"
+  ],
+  "functions": [
+    "link", "rank"
+  ]
+}
+```
+
+## File Structure
+
+Each portable plugin requires the following structure:
+
+- A top-level directory of the name of the plugin. 
+- A json file inside the directory of the name of the plugin.
+- An executable file inside the directory.
+- All other dependencies.
+- Config files (yaml and json) inside 'etc/$pluginType' for each symbol in that plugin.
+
+Take the `car` plugin as an example. To load it automatically, uses need to put it in this structure:
+
+```text
+etc
+  sources
+    json.yaml
+    json.json
+    udp.yaml
+    udp.json
+    sync.yaml
+    sync.json
+  sinks
+    command.json
+  functions
+    link.json
+    rank.json
+plugins
+  portable
+    car
+      server
+      car.json
+```
+
+Notice that, the symbol name must be unique for a specific plugin type. By adding the plugin directory to `plugins/portable`, the plugin will be loaded once eKuiper starts.

sdk/go/api/api.go

@@ -0,0 +1,106 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+)
+
+type SourceTuple interface {
+	Message() map[string]interface{}
+	Meta() map[string]interface{}
+}
+
+type DefaultSourceTuple struct {
+	Mess map[string]interface{} `json:"message"`
+	M    map[string]interface{} `json:"meta"`
+}
+
+func NewDefaultSourceTuple(message map[string]interface{}, meta map[string]interface{}) *DefaultSourceTuple {
+	return &DefaultSourceTuple{
+		Mess: message,
+		M:    meta,
+	}
+}
+
+func (t *DefaultSourceTuple) Message() map[string]interface{} {
+	return t.Mess
+}
+func (t *DefaultSourceTuple) Meta() map[string]interface{} {
+	return t.M
+}
+
+type Source interface {
+	// Open Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext, consumer chan<- SourceTuple, errCh chan<- error)
+	// Configure Called during initialization. Configure the source with the data source(e.g. topic for mqtt) and the properties read from the yaml
+	Configure(datasource string, props map[string]interface{}) error
+	Closable
+}
+
+type Function interface {
+	//The argument is a list of xsql.Expr
+	Validate(args []interface{}) error
+	//Execute the function, return the result and if execution is successful.
+	//If execution fails, return the error and false.
+	Exec(args []interface{}, ctx FunctionContext) (interface{}, bool)
+	//If this function is an aggregate function. Each parameter of an aggregate function will be a slice
+	IsAggregate() bool
+}
+
+type Sink interface {
+	//Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext) error
+	//Called during initialization. Configure the sink with the properties from rule action definition
+	Configure(props map[string]interface{}) error
+	//Called when each row of data has transferred to this sink
+	Collect(ctx StreamContext, data interface{}) error
+	Closable
+}
+
+type Closable interface {
+	Close(ctx StreamContext) error
+}
+
+type Logger interface {
+	Debug(args ...interface{})
+	Info(args ...interface{})
+	Warn(args ...interface{})
+	Error(args ...interface{})
+	Debugln(args ...interface{})
+	Infoln(args ...interface{})
+	Warnln(args ...interface{})
+	Errorln(args ...interface{})
+	Debugf(format string, args ...interface{})
+	Infof(format string, args ...interface{})
+	Warnf(format string, args ...interface{})
+	Errorf(format string, args ...interface{})
+}
+
+type StreamContext interface {
+	context.Context
+	GetLogger() Logger
+	GetRuleId() string
+	GetOpId() string
+	GetInstanceId() int
+	WithMeta(ruleId string, opId string) StreamContext
+	WithInstance(instanceId int) StreamContext
+	WithCancel() (StreamContext, context.CancelFunc)
+}
+
+type FunctionContext interface {
+	StreamContext
+	GetFuncId() int
+}

sdk/go/connection/connection.go

@@ -0,0 +1,174 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package connection
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/context"
+	"go.nanomsg.org/mangos/v3"
+	"go.nanomsg.org/mangos/v3/protocol/pull"
+	"go.nanomsg.org/mangos/v3/protocol/push"
+	"go.nanomsg.org/mangos/v3/protocol/req"
+	_ "go.nanomsg.org/mangos/v3/transport/ipc"
+	"time"
+)
+
+// Options Initialized in plugin.go Start according to the config
+var Options map[string]interface{}
+
+type Closable interface {
+	Close() error
+}
+
+type ReplyFunc func([]byte) []byte
+
+type ControlChannel interface {
+	// reply with string message
+	Run(ReplyFunc) error
+	Closable
+}
+
+type DataInChannel interface {
+	Recv() ([]byte, error)
+	Closable
+}
+
+type DataOutChannel interface {
+	Send([]byte) error
+	Closable
+}
+
+type DataInOutChannel interface {
+	Run(ReplyFunc) error
+	Closable
+}
+
+type NanomsgRepChannel struct {
+	sock mangos.Socket
+}
+
+// Run until process end
+func (r *NanomsgRepChannel) Run(f ReplyFunc) error {
+	err := r.sock.Send([]byte("handshake"))
+	if err != nil {
+		return fmt.Errorf("can't send handshake: %s", err.Error())
+	}
+	for {
+		msg, err := r.sock.Recv()
+		if err != nil {
+			return fmt.Errorf("cannot receive on rep socket: %s", err.Error())
+		}
+		reply := f(msg)
+		err = r.sock.Send(reply)
+		if err != nil {
+			return fmt.Errorf("can't send reply: %s", err.Error())
+		}
+	}
+	return nil
+}
+
+func (r *NanomsgRepChannel) Close() error {
+	return r.sock.Close()
+}
+
+func CreateControlChannel(pluginName string) (ControlChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = req.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new req socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/plugin_%s.ipc", pluginName)
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
+	}
+	return &NanomsgRepChannel{sock: sock}, nil
+}
+
+func CreateSourceChannel(ctx api.StreamContext) (DataOutChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = push.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new push socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on push socket: %s", err.Error())
+	}
+	return sock, nil
+}
+
+func CreateFuncChannel(symbolName string) (DataInOutChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = req.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new req socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/func_%s.ipc", symbolName)
+	if err = sock.Dial(url); err != nil {
+		return nil, fmt.Errorf("can't dial on req socket: %s", err.Error())
+	}
+	return &NanomsgRepChannel{sock: sock}, nil
+}
+
+func CreateSinkChannel(ctx api.StreamContext) (DataInChannel, error) {
+	var (
+		sock mangos.Socket
+		err  error
+	)
+	if sock, err = pull.NewSocket(); err != nil {
+		return nil, fmt.Errorf("can't get new pull socket: %s", err)
+	}
+	setSockOptions(sock)
+	url := fmt.Sprintf("ipc:///tmp/%s_%s_%d.ipc", ctx.GetRuleId(), ctx.GetOpId(), ctx.GetInstanceId())
+	if err = listenWithRetry(sock, url); err != nil {
+		return nil, fmt.Errorf("can't listen on pull socket for %s: %s", url, err.Error())
+	}
+	return sock, nil
+}
+
+func setSockOptions(sock mangos.Socket) {
+	for k, v := range Options {
+		sock.SetOption(k, v)
+	}
+}
+
+func listenWithRetry(sock mangos.Socket, url string) error {
+	var (
+		retryCount    = 300
+		retryInterval = 10
+	)
+	for {
+		err := sock.Listen(url)
+		if err == nil {
+			context.Log.Infof("plugin start to listen after %d tries", retryCount)
+			return err
+		}
+		retryCount--
+		if retryCount < 0 {
+			return err
+		}
+		time.Sleep(time.Duration(retryInterval) * time.Millisecond)
+	}
+}

sdk/go/context/default.go

@@ -0,0 +1,115 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package context
+
+import (
+	"context"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/sirupsen/logrus"
+	"time"
+)
+
+const LoggerKey = "$$logger"
+
+type DefaultContext struct {
+	ruleId     string
+	opId       string
+	instanceId int
+	ctx        context.Context
+	//Only initialized after withMeta set
+	logger api.Logger
+}
+
+func Background() *DefaultContext {
+	c := &DefaultContext{
+		ctx: context.Background(),
+	}
+	return c
+}
+
+func WithValue(parent *DefaultContext, key, val interface{}) *DefaultContext {
+	parent.ctx = context.WithValue(parent.ctx, key, val)
+	return parent
+}
+
+//Implement context interface
+func (c *DefaultContext) Deadline() (deadline time.Time, ok bool) {
+	return c.ctx.Deadline()
+}
+
+func (c *DefaultContext) Done() <-chan struct{} {
+	return c.ctx.Done()
+}
+
+func (c *DefaultContext) Err() error {
+	return c.ctx.Err()
+}
+
+func (c *DefaultContext) Value(key interface{}) interface{} {
+	return c.ctx.Value(key)
+}
+
+// Stream metas
+func (c *DefaultContext) GetContext() context.Context {
+	return c.ctx
+}
+
+func (c *DefaultContext) GetLogger() api.Logger {
+	l, ok := c.ctx.Value(LoggerKey).(*logrus.Entry)
+	if l != nil && ok {
+		return l
+	}
+	return LogEntry("rule", c.ruleId)
+}
+
+func (c *DefaultContext) GetRuleId() string {
+	return c.ruleId
+}
+
+func (c *DefaultContext) GetOpId() string {
+	return c.opId
+}
+
+func (c *DefaultContext) GetInstanceId() int {
+	return c.instanceId
+}
+
+func (c *DefaultContext) WithMeta(ruleId string, opId string) api.StreamContext {
+	return &DefaultContext{
+		ruleId:     ruleId,
+		opId:       opId,
+		instanceId: 0,
+		ctx:        c.ctx,
+	}
+}
+
+func (c *DefaultContext) WithInstance(instanceId int) api.StreamContext {
+	return &DefaultContext{
+		instanceId: instanceId,
+		ruleId:     c.ruleId,
+		opId:       c.opId,
+		ctx:        c.ctx,
+	}
+}
+
+func (c *DefaultContext) WithCancel() (api.StreamContext, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(c.ctx)
+	return &DefaultContext{
+		ruleId:     c.ruleId,
+		opId:       c.opId,
+		instanceId: c.instanceId,
+		ctx:        ctx,
+	}, cancel
+}

sdk/go/context/func_context.go

@@ -0,0 +1,40 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package context
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+)
+
+type DefaultFuncContext struct {
+	api.StreamContext
+	funcId int
+}
+
+func NewDefaultFuncContext(ctx api.StreamContext, id int) *DefaultFuncContext {
+	return &DefaultFuncContext{
+		StreamContext: ctx,
+		funcId:        id,
+	}
+}
+
+func (c *DefaultFuncContext) GetFuncId() int {
+	return c.funcId
+}
+
+func (c *DefaultFuncContext) convertKey(key string) string {
+	return fmt.Sprintf("$$func%d_%s", c.funcId, key)
+}

sdk/go/context/logger.go

@@ -0,0 +1,43 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package context
+
+import (
+	filename "github.com/keepeye/logrus-filename"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	Log *logrus.Logger
+)
+
+func init() {
+	Log = logrus.New()
+	filenameHook := filename.NewHook()
+	filenameHook.Field = "file"
+	Log.AddHook(filenameHook)
+	Log.SetFormatter(&logrus.TextFormatter{
+		TimestampFormat: "2006-01-02 15:04:05",
+		DisableColors:   true,
+		FullTimestamp:   true,
+	})
+	//Log.Level = logrus.DebugLevel
+	Log.WithField("type", "plugin")
+}
+
+func LogEntry(key string, value interface{}) api.Logger {
+	return Log.WithField(key, value)
+}

sdk/go/example/mirror/echo_func.go

@@ -0,0 +1,39 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+)
+
+type echo struct {
+}
+
+func (f *echo) Validate(args []interface{}) error {
+	if len(args) != 1 {
+		return fmt.Errorf("echo function only supports 1 parameter but got %d", len(args))
+	}
+	return nil
+}
+
+func (f *echo) Exec(args []interface{}, _ api.FunctionContext) (interface{}, bool) {
+	result := args[0]
+	return result, true
+}
+
+func (f *echo) IsAggregate() bool {
+	return false
+}

sdk/go/example/mirror/echo_func_test.go

@@ -0,0 +1,54 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/mock"
+	"testing"
+)
+
+func TestValidate(t *testing.T) {
+	f := &echo{}
+	var args []interface{}
+	err := f.Validate(args)
+	if err != nil {
+		exp := "echo function only supports 1 parameter but got 0"
+		if err.Error() != exp {
+			t.Errorf("expect error %s but got %s", exp, err.Error())
+		}
+		return
+	}
+	t.Errorf("should have validation error")
+}
+
+func TestExec(t *testing.T) {
+	var tests = []mock.FuncTest{
+		{
+			Args:   []interface{}{"abc"},
+			Result: "abc",
+			Ok:     true,
+		}, {
+			Args:   []interface{}{1234},
+			Result: 1234,
+			Ok:     true,
+		}, {
+			Args:   []interface{}{[]string{"abc"}},
+			Result: []string{"abc"},
+			Ok:     true,
+		},
+	}
+	f := &echo{}
+	mock.TestFuncExec(f, tests, t)
+}

sdk/go/example/mirror/file_sink.go

@@ -0,0 +1,130 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"os"
+	"sync"
+	"time"
+)
+
+type fileSink struct {
+	interval int
+	path     string
+
+	results [][]byte
+	file    *os.File
+	mux     sync.Mutex
+	cancel  context.CancelFunc
+}
+
+func (m *fileSink) Configure(props map[string]interface{}) error {
+	m.interval = 1000
+	m.path = "cache"
+	if i, ok := props["interval"]; ok {
+		if i, ok := i.(float64); ok {
+			m.interval = int(i)
+		}
+	}
+	if i, ok := props["path"]; ok {
+		if i, ok := i.(string); ok {
+			m.path = i
+		}
+	}
+	return nil
+}
+
+func (m *fileSink) Open(ctx api.StreamContext) error {
+	logger := ctx.GetLogger()
+	logger.Debug("Opening file sink")
+	m.results = make([][]byte, 0)
+	var f *os.File
+	var err error
+	if _, err := os.Stat(m.path); os.IsNotExist(err) {
+		_, err = os.Create(m.path)
+	}
+	f, err = os.OpenFile(m.path, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
+	if err != nil {
+		return fmt.Errorf("fail to open file sink for %v", err)
+	}
+	m.file = f
+	t := time.NewTicker(time.Duration(m.interval) * time.Millisecond)
+	exeCtx, cancel := ctx.WithCancel()
+	m.cancel = cancel
+	go func() {
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				m.save(logger)
+			case <-exeCtx.Done():
+				logger.Info("file sink done")
+				return
+			}
+		}
+	}()
+	return nil
+}
+
+func (m *fileSink) save(logger api.Logger) {
+	if len(m.results) == 0 {
+		return
+	}
+	logger.Debugf("file sink is saving to file %s", m.path)
+	var strings []string
+	m.mux.Lock()
+	for _, b := range m.results {
+		strings = append(strings, string(b)+"\n")
+	}
+	m.results = make([][]byte, 0)
+	m.mux.Unlock()
+	w := bufio.NewWriter(m.file)
+	for _, s := range strings {
+		_, err := m.file.WriteString(s)
+		if err != nil {
+			logger.Errorf("file sink fails to write out result '%s' with error %s.", s, err)
+		}
+	}
+	w.Flush()
+	logger.Debugf("file sink has saved to file %s", m.path)
+}
+
+func (m *fileSink) Collect(ctx api.StreamContext, item interface{}) error {
+	logger := ctx.GetLogger()
+	if v, ok := item.([]byte); ok {
+		logger.Debugf("file sink receive %s", item)
+		m.mux.Lock()
+		m.results = append(m.results, v)
+		m.mux.Unlock()
+	} else {
+		logger.Debug("file sink receive non byte data")
+	}
+	return nil
+}
+
+func (m *fileSink) Close(ctx api.StreamContext) error {
+	if m.cancel != nil {
+		m.cancel()
+	}
+	if m.file != nil {
+		m.save(ctx.GetLogger())
+		return m.file.Close()
+	}
+	return nil
+}

sdk/go/example/mirror/file_sink_test.go

@@ -0,0 +1,58 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bufio"
+	"github.com/lf-edge/ekuiper/sdk/mock"
+	"os"
+	"reflect"
+	"testing"
+)
+
+var CACHE_FILE = "cache"
+
+func TestFileSink(t *testing.T) {
+	defer os.Remove(CACHE_FILE)
+	s := &fileSink{}
+	s.Configure(make(map[string]interface{}))
+	exp := []string{"foo", "bar"}
+	err := mock.RunSinkCollect(s, exp)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+	result := getResults()
+	if !reflect.DeepEqual(result, exp) {
+		t.Errorf("result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", exp, result)
+	}
+}
+
+func getResults() []string {
+	f, err := os.Open(CACHE_FILE)
+	if err != nil {
+		panic(err)
+	}
+	result := make([]string, 0)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		result = append(result, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		panic(err)
+	}
+	_ = f.Close()
+	return result
+}

sdk/go/example/mirror/functions/echo.json

@@ -0,0 +1,27 @@
+{
+	"about": {
+		"trial": false,
+		"author": {
+			"name": "EMQ",
+			"email": "contact@emqx.io",
+			"company": "EMQ Technologies Co., Ltd",
+			"website": "https://www.emqx.io"
+		},
+		"helpUrl": {
+      "en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/plugins/functions/functions.md",
+      "zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/plugins/functions/functions.md"
+		},
+		"description": {
+			"en_US": "",
+			"zh_CN": ""
+		}
+	},
+	"functions": [{
+		"name": "echo",
+		"example": "echo(col1)",
+		"hint": {
+			"en_US": "The parameter value is output as it is.",
+			"zh_CN": "原样输出参数值。"
+		}
+	}]
+}

sdk/go/example/mirror/go.mod

@@ -0,0 +1,10 @@
+module github.com/lf-edge/ekuiper-plugin-mirror
+
+require (
+	github.com/lf-edge/ekuiper/sdk v0.0.0-20210916082120-031cd83a7fd8
+	github.com/mitchellh/mapstructure v1.4.1
+)
+
+replace github.com/lf-edge/ekuiper/sdk => ../../
+
+go 1.16

sdk/go/example/mirror/go.sum

@@ -0,0 +1,299 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
+github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
+github.com/Masterminds/sprig/v3 v3.2.1/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis/v2 v2.15.1/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I=
+github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
+github.com/edgexfoundry/go-mod-core-contracts/v2 v2.0.0/go.mod h1:pfXURRetgIto0GR0sCjDrfa71hqJ1wxmQWi/mOzWfWU=
+github.com/edgexfoundry/go-mod-messaging/v2 v2.0.1/go.mod h1:bLKWB9yeOHLZoQtHLZlGwz8MjsMJIvHDFce7CcUb4fE=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/gdamore/optopia v0.2.0/go.mod h1:YKYEwo5C1Pa617H7NlPcmQXl+vG6YnSSNB44n8dNL0Q=
+github.com/gdexlab/go-render v1.0.1/go.mod h1:wRi5nW2qfjiGj4mPukH4UV0IknS1cHD4VgFTmJX5JzM=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.6.1/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
+github.com/go-redis/redis/v7 v7.3.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU=
+github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag=
+github.com/jhump/protoreflect v1.8.2/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/keepeye/logrus-filename v0.0.0-20190711075016-ce01a4391dd1 h1:JL2rWnBX8jnbHHlLcLde3BBWs+jzqZvOmF+M3sXoNOE=
+github.com/keepeye/logrus-filename v0.0.0-20190711075016-ce01a4391dd1/go.mod h1:nNLjpEi4xVFB7358xLPpPscdvXP+pbhiHgSmjIur8z0=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
+github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
+github.com/lestrrat-go/strftime v1.0.3/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g=
+github.com/lf-edge/ekuiper v0.0.0-20210916082120-031cd83a7fd8 h1:ImaXIkX0lkwN6gW89nWKq3Y4DUmxzwKstYzcVQeiJok=
+github.com/lf-edge/ekuiper v0.0.0-20210916082120-031cd83a7fd8/go.mod h1:kc+6LFOCSu+/uEhkdOO2Fpg5TnaVNm2hs+NlYfZSLmo=
+github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/msgpack-rpc/msgpack-rpc-go v0.0.0-20131026060856-c76397e1782b/go.mod h1:YUWGR1lA7NRnzRjOmYpSgNVip20OCp6NMBwiVFPPhZ8=
+github.com/msgpack/msgpack-go v0.0.0-20130625150338-8224460e6fa3/go.mod h1:jDCQZQaHCHpBYqM4WoGyujFc55bazGAEwK27iK4PQTI=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pebbe/zmq4 v1.2.7/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tebeka/strftime v0.1.5/go.mod h1:29/OidkoWHdEKZqzyDLUyC+LmgDgdHo4WAFCDT7D/Ig=
+github.com/ugorji/go v1.2.5/go.mod h1:gat2tIT8KJG8TVI8yv77nEO/KYT6dV7JE1gfUa8Xuls=
+github.com/ugorji/go/codec v1.2.5/go.mod h1:QPxoTbPKSEAlAHPYt02++xp/en9B/wUdwFCz+hj5caA=
+github.com/urfave/cli v1.22.0/go.mod h1:b3D7uWrF2GilkNgYpgcg6J+JMUw7ehmNkE8sZdliGLc=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
+go.nanomsg.org/mangos/v3 v3.2.1 h1:/7pG6tUJO5ZGznG+waoMy6WrurArODDRJu18848oQnw=
+go.nanomsg.org/mangos/v3 v3.2.1/go.mod h1:RxVwsn46YtfJ74mF8MeVo+MFjg545KCI50NuZrFXmzc=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=

sdk/go/example/mirror/main.go

@@ -0,0 +1,43 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/api"
+	sdk "github.com/lf-edge/ekuiper/sdk/runtime"
+	"os"
+)
+
+func main() {
+	// TODO key case sensitive?
+	sdk.Start(os.Args, &sdk.PluginConfig{
+		Name: "mirror",
+		Sources: map[string]sdk.NewSourceFunc{
+			"random": func() api.Source {
+				return &randomSource{}
+			},
+		},
+		Functions: map[string]sdk.NewFunctionFunc{
+			"echo": func() api.Function {
+				return &echo{}
+			},
+		},
+		Sinks: map[string]sdk.NewSinkFunc{
+			"file": func() api.Sink {
+				return &fileSink{}
+			},
+		},
+	})
+}

sdk/go/example/mirror/mirror.json

@@ -0,0 +1,14 @@
+{
+  "version": "v1.0.0",
+  "language": "go",
+  "executable": "mirror",
+  "sources": [
+    "random"
+  ],
+  "sinks": [
+    "file"
+  ],
+  "functions": [
+    "echo"
+  ]
+}

sdk/go/example/mirror/random_source.go

@@ -0,0 +1,151 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/mitchellh/mapstructure"
+	"math/rand"
+	"strings"
+	"time"
+)
+
+const dedupStateKey = "input"
+
+type randomSourceConfig struct {
+	Interval int                    `json:"interval"`
+	Seed     int                    `json:"seed"`
+	Pattern  map[string]interface{} `json:"pattern"`
+	// how long will the source trace for deduplication. If 0, deduplicate is disabled; if negative, deduplicate will be the whole life time
+	Deduplicate int    `json:"deduplicate"`
+	Format      string `json:"format"`
+}
+
+//Emit data randomly with only a string field
+type randomSource struct {
+	conf *randomSourceConfig
+	list [][]byte
+}
+
+func (s *randomSource) Configure(_ string, props map[string]interface{}) error {
+	cfg := &randomSourceConfig{
+		Format: "json",
+	}
+	err := mapstructure.Decode(props, cfg)
+	if err != nil {
+		return fmt.Errorf("read properties %v fail with error: %v", props, err)
+	}
+	if cfg.Interval <= 0 {
+		return fmt.Errorf("source `random` property `interval` must be a positive integer but got %d", cfg.Interval)
+	}
+	if cfg.Pattern == nil {
+		return fmt.Errorf("source `random` property `pattern` is required")
+	}
+	if cfg.Seed <= 0 {
+		return fmt.Errorf("source `random` property `seed` must be a positive integer but got %d", cfg.Seed)
+	}
+	if strings.ToLower(cfg.Format) != "json" {
+		return fmt.Errorf("random source only supports `json` format")
+	}
+	s.conf = cfg
+	return nil
+}
+
+func (s *randomSource) Open(ctx api.StreamContext, consumer chan<- api.SourceTuple, _ chan<- error) {
+	logger := ctx.GetLogger()
+	logger.Infof("open random source with config %+v", s.conf)
+
+	if s.conf.Deduplicate != 0 {
+		var list interface{}
+		// dedup not supported yet
+		//list, err := ctx.GetState(dedupStateKey)
+		//if err != nil {
+		//	errCh <- err
+		//	return
+		//}
+		if list == nil {
+			list = make([][]byte, 0)
+		} else {
+			if l, ok := list.([][]byte); ok {
+				logger.Debugf("restore list %v", l)
+				s.list = l
+			} else {
+				s.list = make([][]byte, 0)
+				logger.Warnf("random source gets invalid state, ignore it")
+			}
+		}
+	}
+	t := time.NewTicker(time.Duration(s.conf.Interval) * time.Millisecond)
+	defer t.Stop()
+	for {
+		select {
+		case <-t.C:
+			next := randomize(s.conf.Pattern, s.conf.Seed)
+			if s.conf.Deduplicate != 0 && s.isDup(ctx, next) {
+				logger.Debugf("find duplicate")
+				continue
+			}
+			logger.Debugf("Send out data %v", next)
+			consumer <- api.NewDefaultSourceTuple(next, nil)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func randomize(p map[string]interface{}, seed int) map[string]interface{} {
+	r := make(map[string]interface{})
+	for k, v := range p {
+		//TODO other data types
+		vf, ok := v.(float64)
+		if !ok {
+			break
+		}
+		vi := int(vf)
+		r[k] = vi + rand.Intn(seed)
+	}
+	return r
+}
+
+func (s *randomSource) isDup(ctx api.StreamContext, next map[string]interface{}) bool {
+	logger := ctx.GetLogger()
+
+	ns, err := json.Marshal(next)
+	if err != nil {
+		logger.Warnf("invalid input data %v", next)
+		return true
+	}
+	for _, ps := range s.list {
+		if bytes.Compare(ns, ps) == 0 {
+			logger.Debugf("got duplicate %s", ns)
+			return true
+		}
+	}
+	logger.Debugf("no duplicate %s", ns)
+	if s.conf.Deduplicate > 0 && len(s.list) >= s.conf.Deduplicate {
+		s.list = s.list[1:]
+	}
+	s.list = append(s.list, ns)
+	// State not supported yet
+	// ctx.PutState(dedupStateKey, s.list)
+	return false
+}
+
+func (s *randomSource) Close(_ api.StreamContext) error {
+	return nil
+}

sdk/go/example/mirror/random_source_test.go

@@ -0,0 +1,101 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/mock"
+	"reflect"
+	"testing"
+)
+
+func TestConfigure(t *testing.T) {
+	var tests = []struct {
+		p   map[string]interface{}
+		err string
+	}{
+		{
+			p: map[string]interface{}{
+				"interval":    float64(-2),
+				"seed":        float64(1),
+				"pattern":     map[string]interface{}{"count": float64(5)},
+				"deduplicate": 0,
+			},
+			err: "source `random` property `interval` must be a positive integer but got -2",
+		}, {
+			p: map[string]interface{}{
+				"intervala":   float64(-2),
+				"seed":        float64(1),
+				"pattern":     map[string]interface{}{"count": float64(5)},
+				"deduplicate": 0,
+			},
+			err: "source `random` property `interval` must be a positive integer but got 0",
+		}, {
+			p: map[string]interface{}{
+				"interval":    float64(200),
+				"seed":        float64(-1),
+				"pattern":     map[string]interface{}{"count": float64(5)},
+				"deduplicate": 0,
+			},
+			err: "source `random` property `seed` must be a positive integer but got -1",
+		}, {
+			p: map[string]interface{}{
+				"interval":    float64(5),
+				"seed":        float64(5),
+				"deduplicate": 0,
+			},
+			err: "source `random` property `pattern` is required",
+		},
+	}
+	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	for i, tt := range tests {
+		r := &randomSource{}
+		err := r.Configure("new", tt.p)
+		if !reflect.DeepEqual(tt.err, Errstring(err)) {
+			t.Errorf("%d. %q: error mismatch:\n  exp=%s\n  got=%s\n\n", i, err, tt.err, err)
+		}
+	}
+}
+
+func TestRunRandom(t *testing.T) {
+	exp := []api.SourceTuple{
+		api.NewDefaultSourceTuple(map[string]interface{}{"count": 50}, nil),
+		api.NewDefaultSourceTuple(map[string]interface{}{"count": 50}, nil),
+		api.NewDefaultSourceTuple(map[string]interface{}{"count": 50}, nil),
+		api.NewDefaultSourceTuple(map[string]interface{}{"count": 50}, nil),
+		api.NewDefaultSourceTuple(map[string]interface{}{"count": 50}, nil),
+	}
+	p := map[string]interface{}{
+		"interval":    float64(100),
+		"seed":        float64(1), // Use seed = 1, to make sure results always the same
+		"pattern":     map[string]interface{}{"count": float64(50)},
+		"deduplicate": 0,
+	}
+	r := &randomSource{}
+	err := r.Configure("new", p)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+	mock.TestSourceOpen(r, exp, t)
+}
+
+func Errstring(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

sdk/go/example/mirror/sinks/file.json

@@ -0,0 +1,50 @@
+{
+	"about": {
+		"trial": true,
+		"author": {
+			"name": "EMQ",
+			"email": "contact@emqx.io",
+			"company": "EMQ Technologies Co., Ltd",
+			"website": "https://www.emqx.io"
+		},
+		"helpUrl": {
+			"en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/plugins/sinks/file.md",
+			"zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/plugins/sinks/file.md"
+		},
+		"description": {
+			"en_US": "This a sink plugin for file, it can be used for saving the analysis data into file system.",
+			"zh_CN": "本插件为文件持久化插件，可以用于将分析数据存入指定的文件中。"
+		}
+	},
+	"libs": [
+	],
+	"properties": [{
+		"name": "path",
+		"default": "",
+		"optional": false,
+		"control": "text",
+		"type": "string",
+		"hint": {
+			"en_US": "The file path for saving the result",
+			"zh_CN": "保存结果的文件路径"
+		},
+		"label": {
+			"en_US": "Path of file",
+			"zh_CN": "文件路径"
+		}
+	}, {
+		"name": "interval",
+		"default": 1000,
+		"optional": true,
+		"control": "text",
+		"type": "int",
+		"hint": {
+			"en_US": "The time interval (ms) for writing the analysis result.",
+			"zh_CN": "写入分析结果的时间间隔（毫秒）。"
+		},
+		"label": {
+			"en_US": "Intervals",
+			"zh_CN": "间隔时间"
+		}
+	}]
+}

sdk/go/example/mirror/sources/random.json

@@ -0,0 +1,100 @@
+{
+  "about": {
+    "trial": true,
+    "author": {
+      "name": "EMQ",
+      "email": "contact@emqx.io",
+      "company": "EMQ Technologies Co., Ltd",
+      "website": "https://www.emqx.io"
+    },
+    "helpUrl": {
+      "en_US": "https://github.com/lf-edge/ekuiper/blob/master/docs/en_US/plugins/sources/random.md",
+      "zh_CN": "https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/plugins/sources/random.md"
+    },
+    "description": {
+      "en_US": "The source will generate random inputs with a specified pattern.",
+      "zh_CN": "随机源将生成具有指定样式的随机输入。"
+    }
+  },
+  "libs": [],
+  "properties": {
+    "default": [
+      {
+        "name": "interval",
+        "default": 1000,
+        "optional": false,
+        "control": "text",
+        "type": "int",
+        "hint": {
+          "en_US": "The interval (ms) to issue a message",
+          "zh_CN": "发出消息的间隔（毫秒）"
+        },
+        "label": {
+          "en_US": "Interval",
+          "zh_CN": "间隔时间"
+        }
+      },
+      {
+        "name": "seed",
+        "default": 1,
+        "optional": true,
+        "control": "text",
+        "type": "int",
+        "hint": {
+          "en_US": "The maximum integer to be produced by the random function",
+          "zh_CN": "随机函数产生的最大整数"
+        },
+        "label": {
+          "en_US": "Seed",
+          "zh_CN": "最大整数"
+        }
+      },
+      {
+        "name": "deduplicate",
+        "default": 0,
+        "optional": true,
+        "control": "text",
+        "type": "int",
+        "hint": {
+          "en_US": "An int value. If it is a positive number, the source will not issue the messages which are duplicates of any of the previous 'deduplicate' length of messages. If it is 0, the source won't check for duplications. If it is a negative number, the source will check for duplicates over any previous messages. Do not use negative length if you have very large input data sets as all the previous data will be kept.",
+          "zh_CN": "一个整数值。 如果它为正数，则源不会发出与以前任何“重复数据删除”长度的消息重复的消息。如果为0，则源不会检查是否存在重复。如果是负数，则源将检查以前任何消息的重复项。如果有非常大的输入数据集，请不要使用负长度，因为将保留所有以前的数据。"
+        },
+        "label": {
+          "en_US": "Deduplicate",
+          "zh_CN": "去除重复"
+        }
+      },
+      {
+        "name": "pattern",
+        "default": [
+          {
+            "name": "count",
+            "default": 50,
+            "optional": false,
+            "control": "text",
+            "type": "int",
+            "hint": {
+              "en_US": "User-defined fields",
+              "zh_CN": "用户自定义字段名"
+            },
+            "label": {
+              "en_US": "Fields",
+              "zh_CN": "字段名"
+            }
+          }
+        ],
+        "optional": false,
+        "control": "list",
+        "type": "list_object",
+        "hint": {
+          "en_US": "The style generated by the source can define multiple fields. The style is json, for example {\"count\":50}.",
+          "zh_CN": "源生成的样式,可定义多个字段。样式为json，例如{\"count\":50}"
+        },
+        "label": {
+          "en_US": "Pattern",
+          "zh_CN": "样式"
+        }
+      }
+    ]
+  }
+}

sdk/go/example/mirror/sources/random.yaml

@@ -0,0 +1,13 @@
+default:
+  interval: 1000
+  seed: 1
+  pattern:
+    count: 50
+  deduplicate: 0
+
+ext:
+  interval: 100
+
+dedup:
+  interval: 100
+  deduplicate: 50

sdk/go/go.mod

@@ -0,0 +1,14 @@
+module github.com/lf-edge/ekuiper/sdk
+
+go 1.16
+
+require (
+	github.com/keepeye/logrus-filename v0.0.0-20190711075016-ce01a4391dd1
+	github.com/sirupsen/logrus v1.8.1
+	go.nanomsg.org/mangos/v3 v3.2.1
+)
+
+require (
+	github.com/gorilla/websocket v1.4.2 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+)

sdk/go/go.sum

@@ -0,0 +1,22 @@
+github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gdamore/optopia v0.2.0/go.mod h1:YKYEwo5C1Pa617H7NlPcmQXl+vG6YnSSNB44n8dNL0Q=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/keepeye/logrus-filename v0.0.0-20190711075016-ce01a4391dd1 h1:JL2rWnBX8jnbHHlLcLde3BBWs+jzqZvOmF+M3sXoNOE=
+github.com/keepeye/logrus-filename v0.0.0-20190711075016-ce01a4391dd1/go.mod h1:nNLjpEi4xVFB7358xLPpPscdvXP+pbhiHgSmjIur8z0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+go.nanomsg.org/mangos/v3 v3.2.1 h1:/7pG6tUJO5ZGznG+waoMy6WrurArODDRJu18848oQnw=
+go.nanomsg.org/mangos/v3 v3.2.1/go.mod h1:RxVwsn46YtfJ74mF8MeVo+MFjg545KCI50NuZrFXmzc=
+golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

sdk/go/mock/mock.go

@@ -0,0 +1,157 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+	"context"
+	filename "github.com/keepeye/logrus-filename"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/sirupsen/logrus"
+	"time"
+)
+
+type mockContext struct {
+	Ctx    context.Context
+	RuleId string
+	OpId   string
+}
+
+//Implement context interface
+func (c *mockContext) Deadline() (deadline time.Time, ok bool) {
+	return c.Ctx.Deadline()
+}
+
+func (c *mockContext) Done() <-chan struct{} {
+	return c.Ctx.Done()
+}
+
+func (c *mockContext) Err() error {
+	return c.Ctx.Err()
+}
+
+func (c *mockContext) Value(key interface{}) interface{} {
+	return c.Ctx.Value(key)
+}
+
+// Stream metas
+func (c *mockContext) GetContext() context.Context {
+	return c.Ctx
+}
+
+func (c *mockContext) GetLogger() api.Logger {
+	return Logger
+}
+
+func (c *mockContext) GetRuleId() string {
+	return c.RuleId
+}
+
+func (c *mockContext) GetOpId() string {
+	return c.OpId
+}
+
+func (c *mockContext) GetInstanceId() int {
+	return 0
+}
+
+func (c *mockContext) GetRootPath() string {
+	//loc, _ := conf.GetLoc("")
+	return "root path"
+}
+
+func (c *mockContext) SetError(err error) {
+
+}
+
+func (c *mockContext) WithMeta(ruleId string, opId string) api.StreamContext {
+	return c
+}
+
+func (c *mockContext) WithInstance(_ int) api.StreamContext {
+	return c
+}
+
+func (c *mockContext) WithCancel() (api.StreamContext, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(c.Ctx)
+	return &mockContext{
+		RuleId: c.RuleId,
+		OpId:   c.OpId,
+		Ctx:    ctx,
+	}, cancel
+}
+
+func (c *mockContext) IncrCounter(key string, amount int) error {
+	return nil
+}
+
+func (c *mockContext) GetCounter(key string) (int, error) {
+	return 0, nil
+}
+
+func (c *mockContext) PutState(key string, value interface{}) error {
+	return nil
+}
+
+func (c *mockContext) GetState(key string) (interface{}, error) {
+	return nil, nil
+}
+
+func (c *mockContext) DeleteState(key string) error {
+	return nil
+}
+
+func (c *mockContext) Snapshot() error {
+	return nil
+}
+
+func (c *mockContext) SaveState(checkpointId int64) error {
+	return nil
+}
+
+func newMockContext(ruleId string, opId string) api.StreamContext {
+	return &mockContext{Ctx: context.Background(), RuleId: ruleId, OpId: opId}
+}
+
+type mockFuncContext struct {
+	api.StreamContext
+	funcId int
+}
+
+func (fc *mockFuncContext) GetFuncId() int {
+	return fc.funcId
+}
+
+func newMockFuncContext(ctx api.StreamContext, id int) api.FunctionContext {
+	return &mockFuncContext{
+		StreamContext: ctx,
+		funcId:        id,
+	}
+}
+
+var Logger *logrus.Logger
+
+func init() {
+	l := logrus.New()
+	filenameHook := filename.NewHook()
+	filenameHook.Field = "file"
+	l.AddHook(filenameHook)
+	l.SetFormatter(&logrus.TextFormatter{
+		TimestampFormat: "2006-01-02 15:04:05",
+		DisableColors:   true,
+		FullTimestamp:   true,
+	})
+	l.WithField("type", "main")
+	Logger = l
+}

sdk/go/mock/test_func.go

@@ -0,0 +1,39 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"reflect"
+	"testing"
+)
+
+type FuncTest struct {
+	Args   []interface{}
+	Result interface{}
+	Ok     bool
+}
+
+func TestFuncExec(f api.Function, tests []FuncTest, t *testing.T) {
+	ctx := newMockFuncContext(newMockContext("rule1", "op1"), 1)
+	for i, tt := range tests {
+		r, o := f.Exec(tt.Args, ctx)
+		if o != tt.Ok {
+			t.Errorf("%d ok mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.Ok, o)
+		} else if !reflect.DeepEqual(tt.Result, r) {
+			t.Errorf("%d result mismatch:\n\nexp=%#v\n\ngot=%#v\n\n", i, tt.Result, r)
+		}
+	}
+}

sdk/go/mock/test_sink.go

@@ -0,0 +1,32 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import "github.com/lf-edge/ekuiper/sdk/api"
+
+func RunSinkCollect(s api.Sink, exp []string) error {
+	ctx := newMockContext("rule1", "op1")
+	err := s.Open(ctx)
+	if err != nil {
+		return err
+	}
+	for _, e := range exp {
+		err := s.Collect(ctx, []byte(e))
+		if err != nil {
+			return err
+		}
+	}
+	return s.Close(ctx)
+}

sdk/go/mock/test_source.go

@@ -0,0 +1,60 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mock
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestSourceOpen(r api.Source, exp []api.SourceTuple, t *testing.T) {
+	ctx, cancel := newMockContext("rule1", "op1").WithCancel()
+	consumer := make(chan api.SourceTuple)
+	errCh := make(chan error)
+	go r.Open(ctx, consumer, errCh)
+	ticker := time.After(10 * time.Second)
+	limit := len(exp)
+	var result []api.SourceTuple
+outerloop:
+	for {
+		select {
+		case err := <-errCh:
+			t.Errorf("received error: %v", err)
+			cancel()
+			return
+		case tuple := <-consumer:
+			result = append(result, tuple)
+			limit--
+			if limit <= 0 {
+				break outerloop
+			}
+		case <-ticker:
+			t.Errorf("stop after timeout")
+			t.Errorf("expect %v, but got %v", exp, result)
+			cancel()
+			return
+		}
+	}
+	err := r.Close(ctx)
+	if err != nil {
+		t.Errorf(err.Error())
+		return
+	}
+	if !reflect.DeepEqual(exp, result) {
+		t.Errorf("result mismatch:\n  exp=%s\n  got=%s\n\n", exp, result)
+	}
+}

sdk/go/runtime/function.go

@@ -0,0 +1,150 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	context2 "context"
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/connection"
+	"github.com/lf-edge/ekuiper/sdk/context"
+	"sync"
+)
+
+type funcRuntime struct {
+	s      api.Function
+	ch     connection.DataInOutChannel
+	ctx    context2.Context
+	cancel context2.CancelFunc
+	key    string
+}
+
+func setupFuncRuntime(con *Control, s api.Function) (*funcRuntime, error) {
+	// connect to mq server
+	ch, err := connection.CreateFuncChannel(con.SymbolName)
+	if err != nil {
+		return nil, err
+	}
+	context.Log.Info("setup function channel")
+	ctx, cancel := context2.WithCancel(context2.Background())
+	return &funcRuntime{
+		s:      s,
+		ch:     ch,
+		ctx:    ctx,
+		cancel: cancel,
+		key:    fmt.Sprintf("func_%s", con.SymbolName),
+	}, nil
+}
+
+// TODO how to stop? Nearly never end because each function only have one instance
+func (s *funcRuntime) run() {
+	defer s.stop()
+	err := s.ch.Run(func(req []byte) []byte {
+		d := &FuncData{}
+		err := json.Unmarshal(req, d)
+		if err != nil {
+			return encodeReply(false, err)
+		}
+		context.Log.Debugf("running func with %+v", d)
+		switch d.Func {
+		case "Validate":
+			arg, ok := d.Arg.([]interface{})
+			if !ok {
+				return encodeReply(false, "argument is not interface array")
+			}
+			err = s.s.Validate(arg)
+			if err == nil {
+				return encodeReply(true, "")
+			} else {
+				return encodeReply(false, err.Error())
+			}
+		case "Exec":
+			arg, ok := d.Arg.([]interface{})
+			if !ok {
+				return encodeReply(false, "argument is not interface array")
+			}
+			farg, fctx, err := parseFuncContextArgs(arg)
+			if err != nil {
+				return encodeReply(false, err.Error())
+			}
+			r, b := s.s.Exec(farg, fctx)
+			return encodeReply(b, r)
+		case "IsAggregate":
+			result := s.s.IsAggregate()
+			return encodeReply(true, result)
+		default:
+			return encodeReply(false, fmt.Sprintf("invalid func %s", d.Func))
+		}
+	})
+	context.Log.Error(err)
+}
+
+// TODO multiple error
+func (s *funcRuntime) stop() error {
+	s.cancel()
+	err := s.ch.Close()
+	if err != nil {
+		context.Log.Info(err)
+	}
+	context.Log.Info("closed function data channel")
+	reg.Delete(s.key)
+	return nil
+}
+
+func (s *funcRuntime) isRunning() bool {
+	return s.ctx.Err() == nil
+}
+
+func encodeReply(state bool, arg interface{}) []byte {
+	r, _ := json.Marshal(FuncReply{
+		State:  state,
+		Result: arg,
+	})
+	return r
+}
+
+func parseFuncContextArgs(args []interface{}) ([]interface{}, api.FunctionContext, error) {
+	if len(args) < 1 {
+		return nil, nil, fmt.Errorf("exec function context not found")
+	}
+	fargs, temp := args[:len(args)-1], args[len(args)-1]
+	rawCtx, ok := temp.(string)
+	if !ok {
+		return nil, nil, fmt.Errorf("cannot parse function raw context %v", temp)
+	}
+	m := &FuncMeta{}
+	err := json.Unmarshal([]byte(rawCtx), m)
+	if err != nil {
+		return nil, nil, fmt.Errorf("cannot parse function context %v", rawCtx)
+	}
+	if m.RuleId == "" || m.OpId == "" {
+		err := fmt.Sprintf("invalid arg %v, ruleId, opId are required", m)
+		context.Log.Errorf(err)
+		return nil, nil, fmt.Errorf(err)
+	}
+	key := fmt.Sprintf("%s_%s_%d_%d", m.RuleId, m.OpId, m.InstanceId, m.FuncId)
+	if c, ok := exeFuncCtxMap.Load(key); ok {
+		return fargs, c.(api.FunctionContext), nil
+	} else {
+		contextLogger := context.LogEntry("rule", m.RuleId)
+		ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger).WithMeta(m.RuleId, m.OpId)
+		fctx := context.NewDefaultFuncContext(ctx, m.FuncId)
+		exeFuncCtxMap.Store(key, fctx)
+		return fargs, fctx, nil
+	}
+}
+
+var exeFuncCtxMap = &sync.Map{}

sdk/go/runtime/plugin.go

@@ -0,0 +1,214 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Plugin runtime to control the whole plugin with control channel: Distribute symbol data connection, stop symbol and stop plugin
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/connection"
+	"github.com/lf-edge/ekuiper/sdk/context"
+	"go.nanomsg.org/mangos/v3"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+)
+
+var (
+	logger api.Logger
+	reg    runtimes
+)
+
+func initVars(args []string, conf *PluginConfig) {
+	logger = context.LogEntry("plugin", conf.Name)
+	reg = runtimes{
+		content: make(map[string]RuntimeInstance),
+		RWMutex: sync.RWMutex{},
+	}
+	// parse Args
+	if len(args) == 2 {
+		pc := &PortableConfig{}
+		err := json.Unmarshal([]byte(args[1]), pc)
+		if err != nil {
+			panic(fmt.Sprintf("fail to parse args %v", args))
+		}
+		connection.Options = map[string]interface{}{
+			mangos.OptionSendDeadline: pc.SendTimeout,
+		}
+		logger.Infof("config parsed to %v", pc)
+	} else {
+		connection.Options = make(map[string]interface{})
+	}
+}
+
+type NewSourceFunc func() api.Source
+type NewFunctionFunc func() api.Function
+type NewSinkFunc func() api.Sink
+
+// PluginConfig construct once and then read only
+type PluginConfig struct {
+	Name      string
+	Sources   map[string]NewSourceFunc
+	Functions map[string]NewFunctionFunc
+	Sinks     map[string]NewSinkFunc
+}
+
+func (conf *PluginConfig) Get(pluginType string, symbolName string) (builderFunc interface{}) {
+	switch pluginType {
+	case TYPE_SOURCE:
+		if f, ok := conf.Sources[symbolName]; ok {
+			return f
+		}
+	case TYPE_FUNC:
+		if f, ok := conf.Functions[symbolName]; ok {
+			return f
+		}
+	case TYPE_SINK:
+		if f, ok := conf.Sinks[symbolName]; ok {
+			return f
+		}
+	}
+	return nil
+}
+
+// Start Connect to control plane
+// Only run once at process startup
+func Start(args []string, conf *PluginConfig) {
+	initVars(args, conf)
+	logger.Info("starting plugin")
+	ch, err := connection.CreateControlChannel(conf.Name)
+	if err != nil {
+		panic(err)
+	}
+	defer ch.Close()
+	go func() {
+		logger.Info("running control channel")
+		err = ch.Run(func(req []byte) []byte { // not parallel run now
+			c := &Command{}
+			err := json.Unmarshal(req, c)
+			if err != nil {
+				return []byte(err.Error())
+			}
+			logger.Infof("received command %s with arg:'%s'", c.Cmd, c.Arg)
+			ctrl := &Control{}
+			err = json.Unmarshal([]byte(c.Arg), ctrl)
+			if err != nil {
+				return []byte(err.Error())
+			}
+			switch c.Cmd {
+			case CMD_START:
+				f := conf.Get(ctrl.PluginType, ctrl.SymbolName)
+				if f == nil {
+					return []byte("symbol not found")
+				}
+				switch ctrl.PluginType {
+				case TYPE_SOURCE:
+					sf := f.(NewSourceFunc)
+					sr, err := setupSourceRuntime(ctrl, sf())
+					if err != nil {
+						return []byte(err.Error())
+					}
+					go sr.run()
+					regKey := fmt.Sprintf("%s_%s_%d_%s", ctrl.Meta.RuleId, ctrl.Meta.OpId, ctrl.Meta.InstanceId, ctrl.SymbolName)
+					reg.Set(regKey, sr)
+					logger.Infof("running source %s", ctrl.SymbolName)
+				case TYPE_SINK:
+					sf := f.(NewSinkFunc)
+					sr, err := setupSinkRuntime(ctrl, sf())
+					if err != nil {
+						return []byte(err.Error())
+					}
+					go sr.run()
+					regKey := fmt.Sprintf("%s_%s_%d_%s", ctrl.Meta.RuleId, ctrl.Meta.OpId, ctrl.Meta.InstanceId, ctrl.SymbolName)
+					reg.Set(regKey, sr)
+					logger.Infof("running sink %s", ctrl.SymbolName)
+				case TYPE_FUNC:
+					regKey := fmt.Sprintf("func_%s", ctrl.SymbolName)
+					_, ok := reg.Get(regKey)
+					if ok {
+						logger.Infof("got running function instance %s, do nothing", ctrl.SymbolName)
+					} else {
+						ff := f.(NewFunctionFunc)
+						fr, err := setupFuncRuntime(ctrl, ff())
+						if err != nil {
+							return []byte(err.Error())
+						}
+						go fr.run()
+						reg.Set(regKey, fr)
+						logger.Infof("running function %s", ctrl.SymbolName)
+					}
+				default:
+					return []byte(fmt.Sprintf("invalid plugin type %s", ctrl.PluginType))
+				}
+				return []byte(REPLY_OK)
+			case CMD_STOP:
+				// never stop a function symbol here.
+				regKey := fmt.Sprintf("%s_%s_%d_%s", ctrl.Meta.RuleId, ctrl.Meta.OpId, ctrl.Meta.InstanceId, ctrl.SymbolName)
+				logger.Infof("stopping %s", regKey)
+				runtime, ok := reg.Get(regKey)
+				if !ok {
+					return []byte(fmt.Sprintf("symbol %s not found", regKey))
+				}
+				if runtime.isRunning() {
+					err = runtime.stop()
+					if err != nil {
+						return []byte(err.Error())
+					}
+				}
+				return []byte(REPLY_OK)
+			default:
+				return []byte(fmt.Sprintf("invalid command received: %s", c.Cmd))
+			}
+		})
+		if err != nil {
+			logger.Error(err)
+		}
+		os.Exit(1)
+	}()
+	//Stop the whole plugin
+	sigint := make(chan os.Signal, 1)
+	signal.Notify(sigint, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL)
+	<-sigint
+	logger.Infof("stopping plugin %s", conf.Name)
+	os.Exit(0)
+}
+
+// key is rule_op_ins_symbol
+type runtimes struct {
+	content map[string]RuntimeInstance
+	sync.RWMutex
+}
+
+func (r *runtimes) Set(name string, instance RuntimeInstance) {
+	r.Lock()
+	defer r.Unlock()
+	r.content[name] = instance
+}
+
+func (r *runtimes) Get(name string) (RuntimeInstance, bool) {
+	r.RLock()
+	defer r.RUnlock()
+	result, ok := r.content[name]
+	return result, ok
+}
+
+func (r *runtimes) Delete(name string) {
+	r.Lock()
+	defer r.Unlock()
+	delete(r.content, name)
+}

sdk/go/runtime/shared.go

@@ -0,0 +1,68 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+const (
+	TYPE_SOURCE = "source"
+	TYPE_SINK   = "sink"
+	TYPE_FUNC   = "func"
+)
+
+type Meta struct {
+	RuleId     string `json:"ruleId"`
+	OpId       string `json:"opId"`
+	InstanceId int    `json:"instanceId"`
+}
+
+type FuncMeta struct {
+	Meta
+	FuncId int `json:"funcId"`
+}
+
+type Control struct {
+	SymbolName string                 `json:"symbolName"`
+	Meta       *Meta                  `json:"meta,omitempty"`
+	PluginType string                 `json:"pluginType"`
+	DataSource string                 `json:"dataSource,omitempty"`
+	Config     map[string]interface{} `json:"config,omitempty"`
+}
+
+type Command struct {
+	Cmd string `json:"cmd"`
+	Arg string `json:"arg"`
+}
+
+const (
+	CMD_START = "start"
+	CMD_STOP  = "stop"
+)
+
+const (
+	REPLY_OK = "ok"
+)
+
+type PortableConfig struct {
+	SendTimeout int64 `json:"sendTimeout"`
+}
+
+type FuncData struct {
+	Func string      `json:"func"`
+	Arg  interface{} `json:"arg"`
+}
+
+type FuncReply struct {
+	State  bool        `json:"state"`
+	Result interface{} `json:"result"`
+}

sdk/go/runtime/sink.go

@@ -0,0 +1,93 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	context2 "context"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/connection"
+)
+
+type sinkRuntime struct {
+	s      api.Sink
+	ch     connection.DataInChannel
+	ctx    api.StreamContext
+	cancel context2.CancelFunc
+	key    string
+}
+
+func setupSinkRuntime(con *Control, s api.Sink) (*sinkRuntime, error) {
+	ctx, err := parseContext(con)
+	if err != nil {
+		return nil, err
+	}
+	err = s.Configure(con.Config)
+	if err != nil {
+		return nil, err
+	}
+	ch, err := connection.CreateSinkChannel(ctx)
+	if err != nil {
+		return nil, err
+	}
+	ctx.GetLogger().Info("Setup message pipeline, start listening")
+	ctx, cancel := ctx.WithCancel()
+	return &sinkRuntime{
+		s:      s,
+		ch:     ch,
+		ctx:    ctx,
+		cancel: cancel,
+		key:    fmt.Sprintf("%s_%s_%d_%s", con.Meta.RuleId, con.Meta.OpId, con.Meta.InstanceId, con.SymbolName),
+	}, nil
+}
+
+func (s *sinkRuntime) run() {
+	err := s.s.Open(s.ctx)
+	if err != nil {
+		s.stop()
+		return
+	}
+	for {
+		var msg []byte
+		msg, err = s.ch.Recv()
+		if err != nil {
+			s.ctx.GetLogger().Errorf("cannot receive from mangos Socket: %s", err.Error())
+			s.stop()
+			return
+		}
+		err = s.s.Collect(s.ctx, msg)
+		if err != nil {
+			s.ctx.GetLogger().Errorf("collect error: %s", err.Error())
+			s.stop()
+			return
+		}
+	}
+}
+
+func (s *sinkRuntime) stop() error {
+	s.cancel()
+	s.s.Close(s.ctx)
+	err := s.ch.Close()
+	if err != nil {
+		s.ctx.GetLogger().Info(err)
+	}
+	s.ctx.GetLogger().Info("closed sink data channel")
+	reg.Delete(s.key)
+	return nil
+}
+
+func (s *sinkRuntime) isRunning() bool {
+	return s.ctx.Err() == nil
+}

sdk/go/runtime/source.go

@@ -0,0 +1,96 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package runtime
+
+import (
+	context2 "context"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/connection"
+)
+
+// lifecycle controlled by plugin
+// if stop by error, inform plugin
+
+type sourceRuntime struct {
+	s      api.Source
+	ch     connection.DataOutChannel
+	ctx    api.StreamContext
+	cancel context2.CancelFunc
+	key    string
+}
+
+func setupSourceRuntime(con *Control, s api.Source) (*sourceRuntime, error) {
+	// init context with args
+	ctx, err := parseContext(con)
+	// TODO check cmd error handling or using health check
+	if err != nil {
+		return nil, err
+	}
+	// init config with args and call source config
+	err = s.Configure(con.DataSource, con.Config)
+	if err != nil {
+		return nil, err
+	}
+	// connect to mq server
+	ch, err := connection.CreateSourceChannel(ctx)
+	if err != nil {
+		return nil, err
+	}
+	ctx.GetLogger().Info("Setup message pipeline, start sending")
+	ctx, cancel := ctx.WithCancel()
+	return &sourceRuntime{
+		s:      s,
+		ch:     ch,
+		ctx:    ctx,
+		cancel: cancel,
+		key:    fmt.Sprintf("%s_%s_%d_%s", con.Meta.RuleId, con.Meta.OpId, con.Meta.InstanceId, con.SymbolName),
+	}, nil
+}
+
+func (s *sourceRuntime) run() {
+	errCh := make(chan error)
+	consumer := make(chan api.SourceTuple)
+	go s.s.Open(s.ctx, consumer, errCh)
+	for {
+		select {
+		case err := <-errCh:
+			s.ctx.GetLogger().Errorf("%v", err)
+			broadcast(s.ctx, s.ch, err)
+			s.stop()
+		case data := <-consumer:
+			s.ctx.GetLogger().Debugf("broadcast data %v", data)
+			broadcast(s.ctx, s.ch, data)
+		case <-s.ctx.Done():
+			s.s.Close(s.ctx)
+			return
+		}
+	}
+}
+
+func (s *sourceRuntime) stop() error {
+	s.cancel()
+	err := s.ch.Close()
+	if err != nil {
+		s.ctx.GetLogger().Info(err)
+	}
+	s.ctx.GetLogger().Info("closed source data channel")
+	reg.Delete(s.key)
+	return nil
+}
+
+func (s *sourceRuntime) isRunning() bool {
+	return s.ctx.Err() == nil
+}

sdk/go/runtime/symbol.go

@@ -0,0 +1,67 @@
+// Copyright 2021 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Runtime for symbol, to establish data connection
+
+package runtime
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/lf-edge/ekuiper/sdk/api"
+	"github.com/lf-edge/ekuiper/sdk/connection"
+	"github.com/lf-edge/ekuiper/sdk/context"
+)
+
+type RuntimeInstance interface {
+	run()
+	stop() error
+	isRunning() bool
+}
+
+func broadcast(ctx api.StreamContext, sock connection.DataOutChannel, data interface{}) {
+	// encode
+	var (
+		result []byte
+		err    error
+	)
+	switch dt := data.(type) {
+	case error:
+		result, err = json.Marshal(fmt.Sprintf("{\"error\":\"%v\"}", dt))
+		if err != nil {
+			ctx.GetLogger().Errorf("%v", err)
+			return
+		}
+	default:
+		result, err = json.Marshal(dt)
+		if err != nil {
+			ctx.GetLogger().Errorf("%v", err)
+			return
+		}
+	}
+	if err = sock.Send(result); err != nil {
+		ctx.GetLogger().Errorf("Failed publishing: %s", err.Error())
+	}
+}
+
+func parseContext(con *Control) (api.StreamContext, error) {
+	if con.Meta.RuleId == "" || con.Meta.OpId == "" {
+		err := fmt.Sprintf("invalid arg %v, ruleId and opId are required", con)
+		context.Log.Errorf(err)
+		return nil, fmt.Errorf(err)
+	}
+	contextLogger := context.LogEntry("rule", con.Meta.RuleId)
+	ctx := context.WithValue(context.Background(), context.LoggerKey, contextLogger).WithMeta(con.Meta.RuleId, con.Meta.OpId)
+	return ctx, nil
+}