doc(portable): add portable plugin doc and refactor extension doc

Signed-off-by: Jiyong Huang <huangjy@emqx.io>
Signed-off-by: Jianxiang Ran <rxan_embedded@163.com>

cmd/kuiper/main.go

@@ -471,7 +471,7 @@ func main() {
 		{
 			Name:    "drop",
 			Aliases: []string{"drop"},
-			Usage:   "drop stream $stream_name | drop table $table_name |drop rule $rule_name | drop plugin $plugin_type $plugin_name -r $stop | drop service $service_name",
+			Usage:   "drop stream $stream_name | drop table $table_name |drop rule $rule_name | drop plugin $plugin_type $plugin_name -s $stop | drop service $service_name",
 			Subcommands: []cli.Command{
 				{
 					Name:  "stream",
@@ -904,8 +904,10 @@ func getPluginType(arg string) (ptype int, err error) {
 		ptype = 1
 	case "function":
 		ptype = 2
+	case "portable":
+		ptype = 3
 	default:
-		err = fmt.Errorf("Invalid plugin type %s, should be \"source\", \"sink\" or \"function\".\n", arg)
+		err = fmt.Errorf("Invalid plugin type %s, should be \"source\", \"sink\", \"function\" or \"portable\".\n", arg)
 	}
 	return
 }

docs/directory.json

@@ -171,24 +171,55 @@
 		{
 			"title": "扩展 Kuiper",
 			"children": [{
-					"title": "Introduction",
-					"path": "extension/overview"
-				},
-				{
-					"title": "函数扩展",
-					"path": "extension/function"
+				"title": "扩展",
+				"path": "extension/overview"
 				},
 				{
-					"title": "Sink/Action 扩展",
-					"path": "extension/sink"
+					"title": "扩展",
+					"children": [
+						{
+							"title": "外部函数",
+							"path": "extension/external/external_func"
+						}
+					]
 				},
 				{
-					"title": "源扩展",
-					"path": "extension/source"
+					"title": "原生插件",
+					"children": [
+						{
+							"title": "函数扩展",
+							"path": "extension/native/function"
+						},
+						{
+							"title": "扩展",
+							"path": "extension/native/overview"
+						},
+						{
+							"title": "目标 （Sink） 扩展",
+							"path": "extension/native/sink"
+						},
+						{
+							"title": "源（ Source ）扩展",
+							"path": "extension/native/source"
+						}
+					]
 				},
 				{
-					"title": "外部函数",
-					"path": "extension/external_func"
+					"title": "Portable",
+					"children": [
+						{
+							"title": "Portable Plugin",
+							"path": "extension/portable/overview"
+						},
+						{
+							"title": "GO SDK for Portable Plugin",
+							"path": "extension/portable/go_sdk"
+						},
+						{
+							"title": "Python SDK for Portable Plugin",
+							"path": "extension/portable/python_sdk"
+						}
+					]
 				}
 			]
 		},
@@ -421,20 +452,51 @@
 					"path": "extension/overview"
 				},
 				{
-					"title": "Function Extension",
-					"path": "extension/function"
-				},
-				{
-					"title": "Sink Extension",
-					"path": "extension/sink"
+					"title": "External",
+					"children": [
+						{
+							"title": "External Function",
+							"path": "extension/external/external_func"
+						}
+					]
 				},
 				{
-					"title": "Source Extension",
-					"path": "extension/source"
+					"title": "Native",
+					"children": [
+						{
+							"title": "Function Extension",
+							"path": "extension/native/function"
+						},
+						{
+							"title": "Native Plugin",
+							"path": "extension/native/overview"
+						},
+						{
+							"title": "Sink Extension",
+							"path": "extension/native/sink"
+						},
+						{
+							"title": "Source Extension",
+							"path": "extension/native/source"
+						}
+					]
 				},
 				{
-					"title": "External Function",
-					"path": "extension/external_func"
+					"title": "Portable",
+					"children": [
+						{
+							"title": "Portable Plugin",
+							"path": "extension/portable/overview"
+						},
+						{
+							"title": "GO SDK for Portable Plugin",
+							"path": "extension/portable/go_sdk"
+						},
+						{
+							"title": "Python SDK for Portable Plugin",
+							"path": "extension/portable/python_sdk"
+						}
+					]
 				}
 			]
 		},

docs/en_US/cli/plugins.md

@@ -48,7 +48,7 @@ To create a function plugin with multiple exported functions, specify the export
 ```
 
 ### parameters
-1. plugin_type: the type of the plugin. Available values are `["source", "sink", "functions"]`
+1. plugin_type: the type of the plugin. Available values are `["source", "sink", "function", "portable"]`
 2. plugin_name: a unique name of the plugin. The name must be the same as the camel case version of the plugin with lowercase first letter. For example, if the exported plugin name is `Random`, then the name of this plugin is `random`.
 3. file: the url of the plugin files. It must be a zip file with: a compiled so file and the yaml file(only required for sources). The name of the files must match the name of the plugin. Please check [Extension](../extension/overview.md) for the naming rule.
 4. functions: only apply to function plugin which exports multiple functions. The property specifies the exported function names.

docs/en_US/extension/external_func.md

@@ -190,7 +190,7 @@ Thus, the google api proto files must be in the imported path. eKuiper already s
 
 In the external service configuration, there are 1 json file and at least 1 schema file(.proto) to define the function mapping. This will define a 3 layer mappings.
 
-1. eKuiper external service layer: it is defined by the file name of the json. It will be used as a key for the external service in the [REST API](../restapi/services.md) for the describe, delete and update of the service as a whole.
+1. eKuiper external service layer: it is defined by the file name of the json. It will be used as a key for the external service in the [REST API](../../restapi/services.md) for the describe, delete and update of the service as a whole.
 2. Interface layer: it is defined in the `interfaces` section of the json file. This is a virtual layer to group functions with the same schemas so that the shared properties such as address, schema file can be specified only once.
 3. eKuiper function layer: it is defined in the proto file as `rpc`. Notice that, the proto rpcs must be defined under a service section in protobuf. There is no restriction for the name of proto service. The function name is the same as the rpc name in the proto by default. But the user can override the mapping name in the json files's interfaces -> functions section.
 
@@ -239,7 +239,7 @@ When eKuiper is started, it will read and register the external service configur
    ```
    Note: After eKuiper is started, it **cannot** automatically load the system by modifying the configuration file. If you need to update dynamically, please use the REST service.
 
-For dynamic registration and management of services, please refer to [External Service Management API](../restapi/services.md).
+For dynamic registration and management of services, please refer to [External Service Management API](../../restapi/services.md).
 
 ## Usage
 

docs/en_US/extension/function.md

@@ -1,6 +1,6 @@
 # Function Extension
 
-In the eKuiper SQL syntax, [many built-in functions](../sqls/built-in_functions.md) are provided to server for various reusable business logic. However, the users still likely need various reusable business logic which are not covered by the built ins. The function extension is presented to customized the functions.
+In the eKuiper SQL syntax, [many built-in functions](../../sqls/built-in_functions.md) are provided to server for various reusable business logic. However, the users still likely need various reusable business logic which are not covered by the built ins. The function extension is presented to customized the functions.
 
 ## Developing
 
@@ -8,7 +8,7 @@ In the eKuiper SQL syntax, [many built-in functions](../sqls/built-in_functions.
 
 To develop a function for eKuiper is to implement [api.Function](https://github.com/lf-edge/ekuiper/blob/master/pkg/api/stream.go) interface and export it as a golang plugin.
 
-Before starting the development, you must [setup the environment for golang plugin](overview.md#setup-the-plugin-developing-environment). 
+Before starting the development, you must [setup the environment for golang plugin](../overview.md#setup-the-plugin-developing-environment). 
 
 To develop a function, the _Validate_ method is firstly to be implemented. This method will be called during SQL validation. In this method, a slice of [xsql.Expr](https://github.com/lf-edge/ekuiper/blob/master/pkg/ast/expr.go) is passed as the parameter that contains the arguments for this function in the runtime. The developer can do a validation against it to check the argument count and type etc. If validation is successful, return nil. Otherwise, return an error object.
 
@@ -30,7 +30,7 @@ The main task for a Function is to implement _exec_ method. The method will be l
 Exec(args []interface{}) (interface{}, bool)
 ```  
 
-As the function itself is a plugin, it must be in the main package. Given the function struct name is myFunction. At last of the file, the source must be exported as a symbol as below. There are [2 types of exported symbol supported](overview.md#plugin-development). For function extension, if there is no internal state, it is recommended to export a singleton instance.
+As the function itself is a plugin, it must be in the main package. Given the function struct name is myFunction. At last of the file, the source must be exported as a symbol as below. There are [2 types of exported symbol supported](../overview.md#plugin-development). For function extension, if there is no internal state, it is recommended to export a singleton instance.
 
 ```go
 var MyFunction myFunction
@@ -64,8 +64,8 @@ go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/functio
 
 eKuiper will load plugins in the plugin folders automatically. The auto loaded function plugin assumes there is a function named the same as the plugin name. If multiple functions are exported, users need to explicitly register them to make them available. There are two ways to register the functions.
 
-1. In development environment, we recommend to build plugin .so file directly into the plugin folder so that eKuiper can auto load it. Then call [CLI register functions command](../cli/plugins.md#register-functions) or [REST register functions API](../restapi/plugins.md#register-functions).
-2. In production environment, [package the plugin into zip file](../plugins/plugins_tutorial.md#plugin-deployment-1), then call [CLI function plugin create command](../cli/plugins.md#create-a-plugin) or [REST function plugin create API](../restapi/plugins.md#create-a-plugin) with functions list specified.
+1. In development environment, we recommend to build plugin .so file directly into the plugin folder so that eKuiper can auto load it. Then call [CLI register functions command](../../cli/plugins.md#register-functions) or [REST register functions API](../../restapi/plugins.md#register-functions).
+2. In production environment, [package the plugin into zip file](../../plugins/plugins_tutorial.md#plugin-deployment-1), then call [CLI function plugin create command](../../cli/plugins.md#create-a-plugin) or [REST function plugin create API](../../restapi/plugins.md#create-a-plugin) with functions list specified.
 
 ## Usage
 

docs/en_US/extension/native/overview.md

@@ -0,0 +1,55 @@
+# Native Plugin
+
+eKuiper allows user to customize the different kinds of extensions by the native golang plugin system. 
+
+- The source extension is used for extending different stream source, such as consuming data from other message brokers. eKuiper has built-in source support for [MQTT broker](../../rules/sources/mqtt.md).
+- Sink/Action extension is used for extending pub/push data to different targets, such as database, other message system, web interfaces or file systems. Built-in action is supported in eKuiper, see [MQTT](../../rules/sinks/mqtt.md) & [log files](../../rules/sinks/logs.md).
+- Functions extension allows user to extend different functions that used in SQL. Built-in functions is supported in eKuiper, see [functions](../../sqls/built-in_functions.md).
+
+Please read the following to learn how to implement different extensions.
+
+- [Source extension](./source.md)
+- [Sink/Action extension](./sink.md)
+- [Function extension](./function.md)
+
+## Naming
+
+We recommend plugin name to be camel case. Notice that, there are some restrictions for the names:
+
+1. The name of the export symbol of the plugin should be camel case with an **upper case first letter**. It must be the same as the plugin name except the first letter. For example, plugin name _file_ must export a export symbol name _File_ .
+2. The name of _.so_ file must be the same as the export symbol name or the plugin name. For example, _MySource.so_ or _mySink.so_.
+
+## State storage
+
+eKuiper extension exposes a key value state storage interface through the context parameter, which can be used for all types of extensions, including Source/Sink/Function extensions.
+
+States are key-value pairs, where the key is a string and the value is arbitrary data. Keys are scoped the to current extended instance.
+
+Users can access the state storage through the context object. State-related methods include putState, getState, incrCounter, getCounter and deleteState.
+
+Below is an example of a function extension to access states. This function will count the number of words passed in and save the cumulative number in the state.
+
+```go
+func (f *accumulateWordCountFunc) Exec(args []interface{}, ctx api.FunctionContext) (interface{}, bool) {
+    logger := ctx.GetLogger()    
+	err := ctx.IncrCounter("allwordcount", len(strings.Split(args[0], args[1])))
+	if err != nil {
+		return err, false
+	}
+	if c, err := ctx.GetCounter("allwordcount"); err != nil   {
+		return err, false
+	} else {
+		return c, true
+	}
+}
+```
+
+## Runtime dependencies
+
+Some plugin may need to access dependencies in the file system. Those files is put under {{eKuiperPath}}/etc/{{pluginType}}/{{pluginName}} directory. When packaging the plugin, put those files in [etc directory](../../restapi/plugins.md#plugin-file-format). After installation, they will be moved to the recommended place.
+
+In the plugin source code, developers can access the dependencies of file system by getting the eKuiper root path from the context:
+
+```go
+ctx.GetRootPath()
+```

docs/en_US/extension/sink.md

@@ -1,6 +1,6 @@
 # Sink Extension
 
-Sink feed data from eKuiper into external systems. eKuiper has built-in sink support for [MQTT broker](../rules/sinks/mqtt.md) and [log sink](../rules/sinks/logs.md). There are still needs to publish data to various external systems include messaging systems and database etc. Sink extension is presented to meet this requirement.
+Sink feed data from eKuiper into external systems. eKuiper has built-in sink support for [MQTT broker](../../rules/sinks/mqtt.md) and [log sink](../../rules/sinks/logs.md). There are still needs to publish data to various external systems include messaging systems and database etc. Sink extension is presented to meet this requirement.
 
 ## Developing
 
@@ -8,9 +8,9 @@ Sink feed data from eKuiper into external systems. eKuiper has built-in sink sup
 
 To develop a sink for eKuiper is to implement [api.Sink](https://github.com/lf-edge/ekuiper/blob/master/pkg/api/stream.go) interface and export it as a golang plugin.
 
-Before starting the development, you must [setup the environment for golang plugin](overview.md#setup-the-plugin-developing-environment). 
+Before starting the development, you must [setup the environment for golang plugin](../overview.md#setup-the-plugin-developing-environment). 
 
-To develop a sink, the _Configure_ method must be implemented. This method will be called once the sink is initialized. In this method, a map that contains the configuration in the [rule actions definition](../rules/overview.md#actions) is passed in. Typically, there will be information such as host, port, user and password of the external system. You can use this map to initialize this sink.
+To develop a sink, the _Configure_ method must be implemented. This method will be called once the sink is initialized. In this method, a map that contains the configuration in the [rule actions definition](../../rules/overview.md#actions) is passed in. Typically, there will be information such as host, port, user and password of the external system. You can use this map to initialize this sink.
 
 ```go
 //Called during initialization. Configure the sink with the properties from action definition 
@@ -35,7 +35,7 @@ The last method to implement is _Close_ which literally close the connection. It
 Close(ctx StreamContext) error
 ```
 
-As the sink itself is a plugin, it must be in the main package. Given the sink struct name is mySink. At last of the file, the sink must be exported as a symbol as below. There are [2 types of exported symbol supported](overview.md#plugin-development). For sink extension, states are usually needed, so it is recommended to export a constructor function.
+As the sink itself is a plugin, it must be in the main package. Given the sink struct name is mySink. At last of the file, the sink must be exported as a symbol as below. There are [2 types of exported symbol supported](../overview.md#plugin-development). For sink extension, states are usually needed, so it is recommended to export a constructor function.
 
 ```go
 func MySink() api.Sink {
@@ -54,7 +54,7 @@ go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/sink
 
 ### Usage
 
-The customized sink is specified in a [actions definition](../rules/overview.md#actions). Its name is used as the key of the action. The configuration is the value.
+The customized sink is specified in a [actions definition](../../rules/overview.md#actions). Its name is used as the key of the action. The configuration is the value.
 
 If you have developed a sink implementation MySink, you should have:
 1. In the plugin file, symbol MySink is exported.

docs/en_US/extension/portable/go_sdk.md

@@ -0,0 +1,94 @@
+# GO SDK for Portable Plugin
+
+By using GO SDK for portable plugins, user can develop portable plugins with go language. The GO SDK provides similar APIs for the source, sink and function extensions. Additionally, it provides a sdk start function as the execution entry point to define the plugin and its symbols.
+
+## Development
+
+### Symbols
+
+As the GO SDK provides almost identical API interfaces, the user's source, sink and function plugin can almost reuse by only some small modifications.
+
+To develop the portable plugin, users need to depend on `github.com/lf-edge/ekuiper/sdk` instead of eKuiper main project. Then to implement source, just implement the interfaces in package `github.com/lf-edge/ekuiper/sdk/api`.
+
+For source, implement the source interface as below as the same as described in [native plugin source](../native/source.md).
+
+```go
+type Source interface {
+	// Open Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext, consumer chan<- SourceTuple, errCh chan<- error)
+	// Configure Called during initialization. Configure the source with the data source(e.g. topic for mqtt) and the properties read from the yaml
+	Configure(datasource string, props map[string]interface{}) error
+	Closable
+}
+```
+
+For sink, implement the sink interface as below as the same as described in [native plugin sink](../native/sink.md).
+
+```go
+type Sink interface {
+	//Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext) error
+	//Called during initialization. Configure the sink with the properties from rule action definition
+	Configure(props map[string]interface{}) error
+	//Called when each row of data has transferred to this sink
+	Collect(ctx StreamContext, data interface{}) error
+	Closable
+}
+```
+
+For function, implement the function interface as below as the same as described in [native plugin function](../native/function.md).
+
+```go
+type Function interface {
+	//The argument is a list of xsql.Expr
+	Validate(args []interface{}) error
+	//Execute the function, return the result and if execution is successful.
+	//If execution fails, return the error and false.
+	Exec(args []interface{}, ctx FunctionContext) (interface{}, bool)
+	//If this function is an aggregate function. Each parameter of an aggregate function will be a slice
+	IsAggregate() bool
+}
+```
+
+### Plugin Main Program
+
+As the portable plugin is a standalone program, it needs a main program to be able to built into an executable. In go SDK, a start function is provided to define the meta data of the plugin and let it start. A typical main program is as below:
+
+```go
+package main
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/api"
+	sdk "github.com/lf-edge/ekuiper/sdk/runtime"
+	"os"
+)
+
+func main() {
+	sdk.Start(os.Args, &sdk.PluginConfig{
+		Name: "mirror",
+		Sources: map[string]sdk.NewSourceFunc{
+			"random": func() api.Source {
+				return &randomSource{}
+			},
+		},
+		Functions: map[string]sdk.NewFunctionFunc{
+			"echo": func() api.Function {
+				return &echo{}
+			},
+		},
+		Sinks: map[string]sdk.NewSinkFunc{
+			"file": func() api.Sink {
+				return &fileSink{}
+			},
+		},
+	})
+}
+```
+
+Here, in the main function, it calls sdk.Start to start the plugin process. In the argument, a PluginConfig struct is specified to define the plugin name, the sources, functions and sinks name and their initialization functions. This information must match the json file when packaging the plugin.
+
+For the full examples, please check the sdk [example](https://github.com/lf-edge/ekuiper/tree/master/sdk/go/example/mirror).
+
+## Package
+
+We need to prepare the executable file and the json file and then package them. For GO SDK, we need to build the main program into an executable by merely using `go build` like a normal program (it is actually a normal program). Due to go binary file may have different binary name in different os, make sure the file name is correct in the json file. For detail, please check [packaing](./overview.md#package).

docs/en_US/extension/portable/overview.md

@@ -0,0 +1,89 @@
+# Portable Plugin
+
+As a supplement to the native plugin, portable plugins aims to provide the equal functionality while allow to run in more general environments and created by more languages. Similar to native plugins, portable plugins also support to customize source, sink and function extensions.
+
+The steps to create plugin is similar to the native plugin.
+
+1. Develop the plugin with SDK.
+    1. Develop each plugin symbol(source, sink and function) by implementing corresponding interfaces
+    2. Develop the main program to serve all the symbols as one plugin
+2. Build or package the plugin depending on the programing language.
+3. Register the plugin by eKuiper file/REST/CLI.
+
+We aim to provide SDK for all mainstream language. Currently, [go SDK](go_sdk.md) and [python SDK](python_sdk.md) are supported.
+
+## Development
+
+Unlike the native plugin, a portable plugin can bundle multiple *symbols*. Each symbol represents an extension of source, sink or function. The implementation of a symbol is to implement the interface of source, sink or function similar to the native plugin. In portable plugin mode, it is to implement the interface with the selected language. 
+
+Then, the user need to create a main program to define and serve all the symbols. The main program will be run when starting the plugin. The development varies for languages, please check [go SDK](go_sdk.md) and [python SDK](python_sdk.md) for the detail.
+
+### Debugging
+
+We provide a portable plugin test server to simulate the eKuiper main program part while the developers can start the plugin side manually to support debug.
+
+You can find the tool in `tools/plugin_test_server`. It only supports to test a single plugin Testing process.
+0. Edit the testingPlugin variable to match your plugin meta.
+1. Start this server, and wait for handshake.
+2. Start or debug your plugin. Make sure the handshake completed.
+3. Issue startSymbol/stopSymbol REST API  to debug your plugin symbol. The REST API is like:
+   ```
+   POST http://localhost:33333/symbol/start
+   Content-Type: application/json
+   
+   {
+     "symbolName": "pyjson",
+     "meta": {
+       "ruleId": "rule1",
+       "opId": "op1",
+       "instanceId": 1
+     },
+     "pluginType": "source",
+     "config": {}
+   }
+   ```
+
+## Package
+
+After development, we need to package the result into a zip to be installed. Inside the zip file, the file structure must follow this convention with the correct naming:
+
+- {pluginName}.json: the file name must be the same as the plugin name which defines in the plugin main program and the REST/CLI command.
+- the executable file for the plugin main program
+- sources/sinks/functions directory: hold the json or yaml file for all defined symbols by category
+
+Optionally, we can package the supportive files like `install.sh` and the dependencies.
+
+In the json file, we need to depict the metadata of this plugin. The information must match the definitions in the plugin main program. Below is an example:
+
+```json
+{
+  "version": "v1.0.0",
+  "language": "go",
+  "executable": "mirror",
+  "sources": [
+    "random"
+  ],
+  "sinks": [
+    "file"
+  ],
+  "functions": [
+    "echo"
+  ]
+}
+```
+
+A plugin can contain multiple sources, sinks and functions, define them in the corresponding arrays in the json file. A plugin must be implemented in a single language, and specify that in the *language* field. Additionally, the *executable* field is required to specify the plugin main program executable. Please refer to [mirror.zip](https://github.com/lf-edge/ekuiper/blob/master/internal/plugin/testzips/portables/mirror.zip) as an example.
+
+## Management
+
+The portable plugins can be automatically loaded in start up by putting the content(the json, the executable and all supportive files) inside `plugins/portables/${pluginName}` and the configurations to the corresponding directories under `etc`.
+
+To manage the portable plugins in runtime, we can use the [REST](../../restapi/plugins.md) or [CLI](../../cli/plugins.md) commands.
+
+## Restrictions
+
+Currently, there are two limitations compared to native plugins:
+
+1. [State](../native/overview.md#state-storage) and Connection API are not supported. Whereas, state is planned to be supported in the future.
+2. In the [Function interface], the arguments cannot be transferred with the AST which means the user cannot validate the argument types. The only validation supported may be the argument count.
+

docs/en_US/extension/portable/python_sdk.md

@@ -0,0 +1,94 @@
+# Python SDK for Portable Plugin
+
+By using Python SDK for portable plugins, user can develop portable plugins with python language. The Python SDK provides APIs for the source, sink and function interfaces. Additionally, it provides a plugin start function as the execution entry point to define the plugin and its symbols.
+
+To run python plugin, there are two prerequisites in the runtime environment:
+1. Install Python 3.x environment.
+2. Install ekuiper package by `pip install ekuiper`.
+
+## Development
+
+The process is the same: develop the symbols and then develop the main program. Python SDK provides the similar source, sink and function interfaces in python language.
+
+Source interface:
+```python
+  class Source(object):
+    """abstract class for eKuiper source plugin"""
+
+    @abstractmethod
+    def configure(self, datasource: str, conf: dict):
+        """configure with the string datasource and conf map and raise error if any"""
+        pass
+
+    @abstractmethod
+    def open(self, ctx: Context):
+        """run continuously and send out the data or error with ctx"""
+        pass
+
+    @abstractmethod
+    def close(self, ctx: Context):
+        """stop running and clean up"""
+        pass
+```
+
+Sink interface:
+```python
+class Sink(object):
+    """abstract class for eKuiper sink plugin"""
+
+    @abstractmethod
+    def configure(self, conf: dict):
+        """configure with conf map and raise error if any"""
+        pass
+
+    @abstractmethod
+    def open(self, ctx: Context):
+        """open connection and wait to receive data"""
+        pass
+
+    @abstractmethod
+    def collect(self, ctx: Context, data: Any):
+        """callback to deal with received data"""
+        pass
+
+    @abstractmethod
+    def close(self, ctx: Context):
+        """stop running and clean up"""
+        pass
+```
+
+Function interface:
+```python
+class Function(object):
+    """abstract class for eKuiper function plugin"""
+
+    @abstractmethod
+    def validate(self, args: List[Any]):
+        """callback to validate against ast args, return a string error or empty string"""
+        pass
+
+    @abstractmethod
+    def exec(self, args: List[Any], ctx: Context) -> Any:
+        """callback to do execution, return result"""
+        pass
+
+    @abstractmethod
+    def is_aggregate(self):
+        """callback to check if function is for aggregation, return bool"""
+        pass
+```
+
+Users need to create their own source, sink and function by implement these abstract classes. Then create the main program and declare the instantiation functions for these extensions like below:
+
+```python
+if __name__ == '__main__':
+    c = PluginConfig("pysam", {"pyjson": lambda: PyJson()}, {"print": lambda: PrintSink()},
+                     {"revert": lambda: revertIns})
+    plugin.start(c)
+```
+
+For the full example, please check the [python sdk example](https://github.com/lf-edge/ekuiper/tree/master/sdk/python/example/pysam).
+
+## Package
+
+As python is an interpretive language, we don't need to build an executable for it. Just specify the main program python file in the plugin json file is ok. For detail, please check [packaing](./overview.md#package).

docs/en_US/plugins/functions/functions.md

@@ -1,6 +1,6 @@
 # Custom function
 
-eKuiper can customize functions. For the development, compilation and use of functions, please [see here](../../extension/function.md).
+eKuiper can customize functions. For the development, compilation and use of functions, please [see here](../../extension/native/function.md).
 
 ## echo plugin
 

docs/en_US/sqls/overview.md

@@ -6,5 +6,5 @@ eKuiper offers a SQL-like query language for performing transformations and comp
 - [Built-in functions](built-in_functions.md)
 - Extension
   - [Plugin extension](../extension/overview.md)
-  - [External service extension](../extension/external_func.md)
+  - [External service extension](../extension/external/external_func.md)
 

docs/zh_CN/cli/plugins.md

@@ -48,7 +48,7 @@ create plugin $plugin_type $plugin_name $plugin_json | create plugin $plugin_typ
 ```
 
 ### 参数
-1. plugin_type：插件类型，可用值为 `["source", "sink", "functions"]`
+1. plugin_type：插件类型，可用值为 `["source", "sink", "function", "portable"]`
 2. plugin_name：插件的唯一名称。名称首字母必须小写。例如，如果导出的插件名称为 `Random`，则此插件的名称为 `Random`。
 3. file：插件文件的网址。 它必须是一个 zip 文件，其中包含：编译后的 so 文件和 yaml 文件（仅源文件需要）。 文件名称必须与插件名称匹配。 关于命名规则，查看 [扩展名](../extension/overview.md) 。
 4. functions：仅用于导出多个函数的函数插件。该参数指明插件导出的所有函数名。

docs/zh_CN/extension/external_func.md

@@ -189,7 +189,7 @@ import "google/api/annotations.proto";
 
 外部服务配置需要1个 json 文件和至少一个 schema（.proto） 文件。配置定义了服务映射的3个层次。
 
-1. eKuiper 外部服务层: 外部服务名通过 json 文件名定义。这个名字将作为 [REST API](../restapi/services.md) 中描述，删除和更新整体外部服务的键。
+1. eKuiper 外部服务层: 外部服务名通过 json 文件名定义。这个名字将作为 [REST API](../../restapi/services.md) 中描述，删除和更新整体外部服务的键。
 2. 接口层: 定义于 json 文件的 `interfaces` 部分。该层为用户不可见的虚拟层，主要用于将一组服务聚合，以便可以只定义一次一组函数共有的属性，例如 schema，访问地址等。 
 3. eKuiper 函数层: 函数定义于 proto 文件中的`rpc`。需要注意的是，proto 文件中的 `rpc` 必须定义在 proto 文件中的 `service` 之下。此 `sevice` 与 eKuiper 中的外部服务概念不同，且没有关联，其取名没有任何限制。默认情况下，外部函数的名字与 rpc 名字相同。用户可通过修改 json 文件中，interface 下的 functions 部分来覆盖函数名的映射关系。 
 
@@ -233,7 +233,7 @@ eKuiper 启动时，会读取配置文件夹 *etc/services* 里的外部服务
    ```
 注意：eKuiper 启动之后，修改配置文件**不能**自动载入系统。需要动态更新时，请使用 REST 服务。
 
-服务的动态注册和管理，请参考[外部服务管理 API](../restapi/services.md)。
+服务的动态注册和管理，请参考[外部服务管理 API](../../restapi/services.md)。
 
 ## 使用
 

docs/zh_CN/extension/function.md

@@ -1,6 +1,6 @@
 # 函数扩展
 
-在 eKuiper SQL 语法中，向服务器提供了[许多内置函数](../sqls/built-in_functions.md)，用于各种可重用的业务逻辑。 但是，用户仍然可能需要其他未被内置插件覆盖的可重用的业务逻辑。 提供函数扩展是为了自定义函数。
+在 eKuiper SQL 语法中，向服务器提供了[许多内置函数](../../sqls/built-in_functions.md)，用于各种可重用的业务逻辑。 但是，用户仍然可能需要其他未被内置插件覆盖的可重用的业务逻辑。 提供函数扩展是为了自定义函数。
 
 ## 开发
 
@@ -8,7 +8,7 @@
 
 为 eKuiper 开发函数的过程，就是实现 [api.Function](https://github.com/lf-edge/ekuiper/blob/master/pkg/api/stream.go) 接口并将其导出为 golang 插件。
 
-在开始开发之前，您必须为 [golang 插件设置环境](overview.md#setup-the-plugin-developing-environment)。
+在开始开发之前，您必须为 [golang 插件设置环境](../overview.md#setup-the-plugin-developing-environment)。
 
 为了开发函数，首先要实现 _Validate_ 方法。 在 SQL 验证期间将调用此方法。 在此方法中，将传递 [xsql.Expr](https://github.com/lf-edge/ekuiper/blob/master/pkg/ast/expr.go) 的切片作为参数，该参数包含运行时该函数的参数。 开发人员可以对其进行验证，以检查参数计数和类型等。如果验证成功，则返回 nil。 否则，返回一个错误对象。
 
@@ -30,7 +30,7 @@ IsAggregate() bool
 Exec(args []interface{}) (interface{}, bool)
 ```
 
-由于该函数本身是一个插件，因此必须位于 main 程序包中。 给定的函数结构名称为 myFunction。 在文件的最后，必须将源文件作为符号导出，如下所示。 有[2种类型的导出符号被支持](overview.md#plugin-development)。 对于函数扩展，如果没有内部状态，建议导出单例实例。
+由于该函数本身是一个插件，因此必须位于 main 程序包中。 给定的函数结构名称为 myFunction。 在文件的最后，必须将源文件作为符号导出，如下所示。 有[2种类型的导出符号被支持](../overview.md#plugin-development)。 对于函数扩展，如果没有内部状态，建议导出单例实例。
 
 ```go
 var MyFunction myFunction
@@ -63,8 +63,8 @@ go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/functio
 
 eKuiper 启动时会自动载入插件目录里已编译好的插件。自动载入的函数插件假设插件里仅导出一个同名的函数。如果插件导出多个函数，则需要显示运行一次注册操作。有两种方法可以注册函数：
 
-1. 在开发环境中，建议直接构建插件 .so 文件到插件目录中以便 eKuiper 自动载入。构建完成后，运行 [CLI 注册函数命令](../cli/plugins.md#register-functions) or [REST 注册函数 API](../restapi/plugins.md#register-functions) 进行注册。
-2. 在生产环境中，[打包插件到 zip 压缩包](../plugins/plugins_tutorial.md#plugin-deployment-1)，然后运行 [CLI 创建函数插件命令](../cli/plugins.md#create-a-plugin) 或者 [REST 创建函数 API](../restapi/plugins.md#create-a-plugin) 并设置 functions 参数以指定导出函数名。
+1. 在开发环境中，建议直接构建插件 .so 文件到插件目录中以便 eKuiper 自动载入。构建完成后，运行 [CLI 注册函数命令](../../cli/plugins.md#register-functions) or [REST 注册函数 API](../../restapi/plugins.md#register-functions) 进行注册。
+2. 在生产环境中，[打包插件到 zip 压缩包](../../plugins/plugins_tutorial.md#plugin-deployment-1)，然后运行 [CLI 创建函数插件命令](../../cli/plugins.md#create-a-plugin) 或者 [REST 创建函数 API](../../restapi/plugins.md#create-a-plugin) 并设置 functions 参数以指定导出函数名。
 
 ### 使用
 

docs/zh_CN/extension/native/overview.md

@@ -0,0 +1,76 @@
+# 扩展
+
+eKuiper 允许用户自定义扩展，以支持更多的功能。用户可编写插件进行扩展；也可以通过配置的方式，扩展 SQL 中的函数，用于调用外部已有的 REST 或 RPC 服务。
+
+使用插件扩展较为复杂，需要用户编写代码并自行编译，具有一定的开发成本。其使用的场景包括：
+
+- 需要扩展源或是 sink
+- 对性能要求较高
+
+使用外部函数扩展，仅需要进行配置，但其需要通过网络进行调用，有一定性能损耗。使用的场景包括：
+
+- 调用已有的服务，如 REST 或 grpc 提供的 AI 服务
+- 需要灵活部署的服务
+
+## 插件扩展
+
+eKuiper 允许用户自定义不同类型的扩展。 
+
+- 源扩展用于扩展不同的流源，例如使用来自其他消息服务器的数据。eKuiper 对 [MQTT 消息服务器](../../rules/sources/mqtt.md)的内置源提供支持。
+- Sink/Action 扩展用于将发布/推送数据扩展到不同的目标，例如数据库，其他消息系统，Web 界面或文件系统。eKuiper 中提供内置动作支持，请参阅  [MQTT](../../rules/sinks/mqtt.md)  & [日志文件](../../rules/sinks/logs.md).。
+- 函数扩展允许用户扩展 SQL 中使用的不同函数。 eKuiper支持内置函数，请参见 [函数](../../sqls/built-in_functions.md)。
+
+请阅读以下内容，了解如何实现不同的扩展。
+
+- [源扩展](./source.md)
+- [Sink/Action 扩展](./sink.md)
+- [函数扩展](./function.md)
+
+## 命名
+
+建议插件名使用 camel case 形式。插件命名有一些限制：
+1. 插件输出变量必须为**插件名的首字母大写形式**。 例如，插件名为 _file_ ，则其输出变量名必须为 _File_。
+2. _.so_ 文件的名字必须与输出变量名或者插件名相同。例如， _MySource.so_ 或 _mySink.so_。
+
+## 状态存储
+
+eKuiper 扩展通过 context 参数暴露了一个基于键值对的状态存储接口，可用于所有类型的扩展，包括 Source，Sink 和 Function 扩展.
+
+状态为键值对，其中键为 string 类型而值为任意数据。键的作用域仅为当前扩展的实例。
+
+用户可通过 context 对象访问状态存储。状态相关方法包括 putState, getState, incrCounter, getCounter and deleteState。
+
+以下代码为函数扩展访问状态的实例。该函数将计算传入的单词数，并将累积数目保存在状态中。
+
+```go
+func (f *accumulateWordCountFunc) Exec(args []interface{}, ctx api.FunctionContext) (interface{}, bool) {
+    logger := ctx.GetLogger()    
+	err := ctx.IncrCounter("allwordcount", len(strings.Split(args[0], args[1])))
+	if err != nil {
+		return err, false
+	}
+	if c, err := ctx.GetCounter("allwordcount"); err != nil   {
+		return err, false
+	} else {
+		return c, true
+	}
+}
+```
+
+### 运行时依赖
+
+有些插件可能需要访问文件系统中的依赖文件。依赖文件建放置于 {{ekuiperPath}}/etc/{{pluginType}}/{{pluginName}} 目录。打包插件时，依赖文件应放置于 [etc 目录](../../restapi/plugins.md#plugin-file-format)。安装后，这些文件会自动移动到推荐的位置。
+
+在插件源代码中，开发者可通过 context 获取 eKuiper 根目录，以访问文件系统中的依赖：
+
+```go
+ctx.GetRootPath()
+```
+
+## 外部函数扩展
+
+提供一种配置的方式，使得 eKuiper 可以使用 SQL 以函数的方式直接调用外部服务，包括各种 rpc 服务， http 服务等。该方式将可大提高 eKuiper 扩展的易用性。外部函数将作为插件系统的补充，仅在性能要求较高的情况下才建议使用插件。
+
+以 getFeature 函数为例，假设有 AI 服务基于 grpc 提供getFeature 服务。则可在 eKuiper 配置之后，使用 `SELECT getFeature(self) from demo` 的方式，无需定制插件而调用该 AI 服务。
+
+详细配置方法，请参考[外部函数](../external/external_func.md)。

docs/zh_CN/extension/sink.md

@@ -1,6 +1,6 @@
-# Sink （目标） 扩展
+# 目标 （Sink） 扩展
 
-eKuiper 可以将数据接收到外部系统。 eKuiper具有对  [MQTT 消息服务器](../rules/sinks/mqtt.md) 和 [日志目标](../rules/sinks/logs.md)的内置接收器支持。然而， 仍然需要将数据发布到各种外部系统，包括消息传递系统和数据库等。Sink （目标）扩展正是为了满足这一要求。
+eKuiper 可以将数据接收到外部系统。 eKuiper具有对  [MQTT 消息服务器](../../rules/sinks/mqtt.md) 和 [日志目标](../../rules/sinks/logs.md)的内置接收器支持。然而， 仍然需要将数据发布到各种外部系统，包括消息传递系统和数据库等。Sink （目标）扩展正是为了满足这一要求。
 
 ## 开发
 
@@ -8,9 +8,9 @@ eKuiper 可以将数据接收到外部系统。 eKuiper具有对  [MQTT 消息
 
 为 eKuiper 开发 Sink （目标），是实现 [api.Sink](https://github.com/lf-edge/ekuiper/blob/master/pkg/api/stream.go) 接口并将其导出为 golang 插件。
 
-在开始开发之前，您必须为 [golang 插件设置环境](overview.md#setup-the-plugin-developing-environment)。
+在开始开发之前，您必须为 [golang 插件设置环境](../overview.md#setup-the-plugin-developing-environment)。
 
-要开发 Sink （目标），必须实现 _Configure_ 方法。 接收器初始化后，将调用此方法。 在此方法中，将传入包含 [规则操作定义](../rules/overview.md#actions)中的配置映射，通常，将包含诸如外部系统的主机、端口、用户和密码之类的信息。您可以使用此映射来初始化此 Sink （目标）。
+要开发 Sink （目标），必须实现 _Configure_ 方法。 接收器初始化后，将调用此方法。 在此方法中，将传入包含 [规则操作定义](../../rules/overview.md#actions)中的配置映射，通常，将包含诸如外部系统的主机、端口、用户和密码之类的信息。您可以使用此映射来初始化此 Sink （目标）。
 
 ```go
 //Called during initialization. Configure the sink with the properties from action definition 
@@ -36,7 +36,7 @@ Collect(ctx StreamContext, data interface{}) error
 Close(ctx StreamContext) error
 ```
 
-由于 Sink （目标）本身是一个插件，因此它必须位于主程序包中。 给定 Sink （目标）结构名称为 mySink。 在文件的最后，必须将 Sink （目标）导出为以下符号。 共有 [2种类型的导出符号](overview.md#plugin-development)。 对于 Sink （目标）扩展，通常需要状态，因此建议导出构造函数。
+由于 Sink （目标）本身是一个插件，因此它必须位于主程序包中。 给定 Sink （目标）结构名称为 mySink。 在文件的最后，必须将 Sink （目标）导出为以下符号。 共有 [2种类型的导出符号](../overview.md#plugin-development)。 对于 Sink （目标）扩展，通常需要状态，因此建议导出构造函数。
 
 ```go
 func MySink() api.Sink {
@@ -55,7 +55,7 @@ go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/sinks/M
 
 ### 使用
 
-自定义 Sink （目标）在 [动作定义](../rules/overview.md#actions)规定。 它的名称用作操作的键， 配置就是值。
+自定义 Sink （目标）在 [动作定义](../../rules/overview.md#actions)规定。 它的名称用作操作的键， 配置就是值。
 
 如果您开发了 Sink （目标）实现 MySink，则应该具有：
 1. 在插件文件中，将导出符号 MySink。

docs/zh_CN/extension/overview.md

@@ -22,9 +22,9 @@ eKuiper 允许用户自定义不同类型的扩展。
 
 请阅读以下内容，了解如何实现不同的扩展。
 
-- [源扩展](./source.md)
-- [Sink/Action 扩展](./sink.md)
-- [函数扩展](./function.md)
+- [源扩展](native/source.md)
+- [Sink/Action 扩展](native/sink.md)
+- [函数扩展](native/function.md)
 
 ## 命名
 
@@ -73,4 +73,4 @@ ctx.GetRootPath()
 
 以 getFeature 函数为例，假设有 AI 服务基于 grpc 提供getFeature 服务。则可在 eKuiper 配置之后，使用 `SELECT getFeature(self) from demo` 的方式，无需定制插件而调用该 AI 服务。
 
-详细配置方法，请参考[外部函数](external_func.md)。
+详细配置方法，请参考[外部函数](external/external_func.md)。

docs/zh_CN/extension/portable/go_sdk.md

@@ -0,0 +1,91 @@
+# Portable 插件 Go SDK
+
+用户可利用 GO SDK 来开发 portable 插件，这个 SDK 提供了类似原生插件的 API，另外它提供了启动函数，用户只需填充插件信息即可。
+
+## 插件开发
+
+### Symbols
+
+由于 portable 插件 GO SDK 提供了类似原生插件的API，用户做简单的修改即可复用以前编写的原生插件
+
+用户只需依赖 `github.com/lf-edge/ekuiper/sdk` 而不是 eKuiper 主项目即可编写 portable 插件，用户需要实现 `github.com/lf-edge/ekuiper/sdk/api` 中的相应接口即可
+
+对于源，实现跟[原生源插件](../native/source.md)中一样的接口即可 
+
+```go
+type Source interface {
+	// Open Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext, consumer chan<- SourceTuple, errCh chan<- error)
+	// Configure Called during initialization. Configure the source with the data source(e.g. topic for mqtt) and the properties read from the yaml
+	Configure(datasource string, props map[string]interface{}) error
+	Closable
+}
+```
+
+对于目标，实现跟[原生目标插件](../native/sink.md)中一样的接口即可
+
+```go
+type Sink interface {
+	//Should be sync function for normal case. The container will run it in go func
+	Open(ctx StreamContext) error
+	//Called during initialization. Configure the sink with the properties from rule action definition
+	Configure(props map[string]interface{}) error
+	//Called when each row of data has transferred to this sink
+	Collect(ctx StreamContext, data interface{}) error
+	Closable
+}
+```
+
+对于函数，实现跟[原生函数插件](../native/function.md)中一样的接口即可
+
+```go
+type Function interface {
+	//The argument is a list of xsql.Expr
+	Validate(args []interface{}) error
+	//Execute the function, return the result and if execution is successful.
+	//If execution fails, return the error and false.
+	Exec(args []interface{}, ctx FunctionContext) (interface{}, bool)
+	//If this function is an aggregate function. Each parameter of an aggregate function will be a slice
+	IsAggregate() bool
+}
+```
+
+### 插件主程序
+由于 portable 插件是一个独立的程序，需要编写成一个可执行程序。在 GO SDK 中, 提供了启动函数，用户只需填充插件信息即可。启动函数如下：
+
+```go
+package main
+
+import (
+	"github.com/lf-edge/ekuiper/sdk/api"
+	sdk "github.com/lf-edge/ekuiper/sdk/runtime"
+	"os"
+)
+
+func main() {
+	sdk.Start(os.Args, &sdk.PluginConfig{
+		Name: "mirror",
+		Sources: map[string]sdk.NewSourceFunc{
+			"random": func() api.Source {
+				return &randomSource{}
+			},
+		},
+		Functions: map[string]sdk.NewFunctionFunc{
+			"echo": func() api.Function {
+				return &echo{}
+			},
+		},
+		Sinks: map[string]sdk.NewSinkFunc{
+			"file": func() api.Sink {
+				return &fileSink{}
+			},
+		},
+	})
+}
+```
+在主函数中调用了 `sdk.Start`来启动插件进程。在参数中，`PluginConfig` 定义了插件名字，源，目标，函数构造函数。注意这些信息必须跟插件安装包中的 json 描述文件一致
+
+完整例子请参考这个[例子](https://github.com/lf-edge/ekuiper/tree/master/sdk/go/example/mirror)
+
+## 打包发布
+我们需要将可执行文件和 json 描述文件一起打包，使用 GO SDK，仅仅需要 `go build`编译出可执行文件即可。由于在不同操作系统下编译出到的可执行文件名字有所不同，需要确保 json 描述文件中可执行文件名字的准确性。详细信息，请[参考](./overview.md#package)

docs/zh_CN/extension/portable/overview.md

@@ -0,0 +1,81 @@
+# Portable 插件
+作为对原生插件的补充，可移植插件旨在提供相同的功能，同时允许在更通用的环境中运行并由更多语言创建。与原生插件类似，可移植插件也支持自定义 源、目标 和功能扩展。
+
+创建插件的步骤与原生插件类似
+
+1. 使用SDK开发插件。
+   1. 通过实现相应的接口来开发每个插件符号（source、sink和function）
+   2. 开发主程序，将所有交易品种作为一个插件提供服务
+2. 根据编程语言构建或打包插件。
+3. 通过eKuiper文件/REST/CLI注册插件
+
+我们的目标是为所有主流语言提供插件. 当前, [go SDK](go_sdk.md) and [python SDK](python_sdk.md) 已经支持.
+
+## 开发
+
+与原生插件不同，portable 插件可以捆绑多个 *Symbol*。每个 Symbol 代表源、汇或功能的扩展。一个符号的实现就是实现类似于原生插件的source、sink或者function的接口。在 portable 插件模式下，就是用选择的语言来实现接口。
+然后，用户需要创建一个主程序来定义和服务所有的符号。启动插件时将运行主程序。开发因语言而异，详情请查看 [go SDK](go_sdk.md) 和 [python SDK](python_sdk.md)。
+
+### 调试
+我们提供了一个 portable 插件测试服务器来模拟 eKuiper 主程序部分，而开发者可以手动启动插件端以支持调试。
+您可以在`tools/plugin_test_server` 中找到该工具。它只支持测试单个插件测试过程。
+0. 编辑 testingPlugin 变量以匹配您的插件元数据。
+1. 启动此服务器，等待握手。
+2. 启动或调试您的插件。确保握手完成。
+3. 发出 startSymbol/stopSymbol REST API 来调试您的插件符号。 REST API 是这样的：
+   ```
+   POST http://localhost:33333/symbol/start
+   Content-Type: application/json
+   
+   {
+     "symbolName": "pyjson",
+     "meta": {
+       "ruleId": "rule1",
+       "opId": "op1",
+       "instanceId": 1
+     },
+     "pluginType": "source",
+     "config": {}
+   }
+   ```
+
+## 打包发布
+
+开发完成后，我们需要将结果打包成zip进行安装。在 zip 文件中，文件结构必须遵循以下约定并使用正确的命名：
+
+- {pluginName}.json：文件名必须与插件主程序和REST/CLI命令中定义的插件名相同。
+- 插件主程序的可执行文件
+- source/sinks/functions 目录：按类别保存所有已定义符号的 json 或 yaml 文件
+
+或者，我们可以打包其他支持文件，如 `install.sh` 和依赖项。
+
+在json文件中，我们需要描述这个插件的元数据。该信息必须与插件主程序中的定义相匹配。下面是一个例子：
+```json
+{
+  "version": "v1.0.0",
+  "language": "go",
+  "executable": "mirror",
+  "sources": [
+    "random"
+  ],
+  "sinks": [
+    "file"
+  ],
+  "functions": [
+    "echo"
+  ]
+}
+```
+一个插件可以包含多个源、目标和函数，在 json 文件中的相应数组中定义它们。插件必须以单一语言实现，并在 *language* 字段中指定。此外，*executable* 字段需要指定插件主程序可执行文件。请参考[mirror.zip](https://github.com/lf-edge/ekuiper/blob/master/internal/plugin/testzips/portables/mirror.zip) 。
+
+## 管理
+
+通过将内容（json、可执行文件和所有支持文件）放在`plugins/portables/${pluginName}`中，并将配置放在`etc`下的相应目录中，可以在启动时自动加载可移植插件。
+
+要在运行时管理可移植插件，我们可以使用 [REST](../../restapi/plugins.md) 或 [CLI](../../cli/plugins.md) 命令。
+## 限制
+
+目前，与原生插件相比，有两个限制：
+
+1. 不支持 [State](../native/overview.md#state-storage) 和 Connection API。而 state 计划在未来得到支持。
+2. 在函数接口中，参数不能通过AST传递，即用户无法验证参数类型。唯一支持的验证可能是参数计数。

docs/zh_CN/extension/portable/python_sdk.md

@@ -0,0 +1,93 @@
+# Portable 插件 Python SDK 
+用户可利用 Python SDK 来开发 portable 插件，这个 SDK 提供了类似原生插件的 API，另外它提供了启动函数，用户只需填充插件信息即可。
+
+为了运行 python 插件，有两个前置条件
+To run python plugin, there are two prerequisites in the runtime environment:
+1. 安装 Python 3.x 环境.
+2. 通过 `pip install ekuiper` 安装 ekuiper 包.
+
+## 插件开发
+
+开发插件包括子模块和主程序两部分, Python SDK 提供了 python 语言的源，目标和函数 API。
+
+源接口:
+```python
+  class Source(object):
+    """abstract class for eKuiper source plugin"""
+
+    @abstractmethod
+    def configure(self, datasource: str, conf: dict):
+        """configure with the string datasource and conf map and raise error if any"""
+        pass
+
+    @abstractmethod
+    def open(self, ctx: Context):
+        """run continuously and send out the data or error with ctx"""
+        pass
+
+    @abstractmethod
+    def close(self, ctx: Context):
+        """stop running and clean up"""
+        pass
+```
+
+目标接口:
+```python
+class Sink(object):
+    """abstract class for eKuiper sink plugin"""
+
+    @abstractmethod
+    def configure(self, conf: dict):
+        """configure with conf map and raise error if any"""
+        pass
+
+    @abstractmethod
+    def open(self, ctx: Context):
+        """open connection and wait to receive data"""
+        pass
+
+    @abstractmethod
+    def collect(self, ctx: Context, data: Any):
+        """callback to deal with received data"""
+        pass
+
+    @abstractmethod
+    def close(self, ctx: Context):
+        """stop running and clean up"""
+        pass
+```
+
+函数接口:
+```python
+class Function(object):
+    """abstract class for eKuiper function plugin"""
+
+    @abstractmethod
+    def validate(self, args: List[Any]):
+        """callback to validate against ast args, return a string error or empty string"""
+        pass
+
+    @abstractmethod
+    def exec(self, args: List[Any], ctx: Context) -> Any:
+        """callback to do execution, return result"""
+        pass
+
+    @abstractmethod
+    def is_aggregate(self):
+        """callback to check if function is for aggregation, return bool"""
+        pass
+```
+用户通过实现这些抽象接口来创建自己的源，目标和函数，然后在主函数中声明这些自定义插件的实例化方法
+
+```python
+if __name__ == '__main__':
+    c = PluginConfig("pysam", {"pyjson": lambda: PyJson()}, {"print": lambda: PrintSink()},
+                     {"revert": lambda: revertIns})
+    plugin.start(c)
+```
+
+关于更详细的信息，请参考这篇文章 [python sdk example](https://github.com/lf-edge/ekuiper/tree/master/sdk/python).
+
+## 打包发布
+
+由于 python 是解释性语言，不需要编译出可执行文件，需要确保 json 描述文件中可执行文件名字的准确性即可。详细信息，请[参考](./overview.md#package)

docs/zh_CN/manager-ui/plugins_in_manager.md

@@ -67,7 +67,7 @@ Source 元数据的配置参数主要由两部分组成：
 
 **properties**
 
-描述插件的可配置属性信息，包括参数的信息以及如何进行界面展示。Source 的属性信息通过对应的配置文件指定，且配置文件中可指定多个配置组，详细请见[Source 配置文档](../extension/source.md#处理配置)。在元数据文件中，`properties`下可对应有多个配置组名，如例子中的`default`；每个配置组下又有多个属性的元数据。
+描述插件的可配置属性信息，包括参数的信息以及如何进行界面展示。Source 的属性信息通过对应的配置文件指定，且配置文件中可指定多个配置组，详细请见[Source 配置文档](../extension/native/source.md#处理配置)。在元数据文件中，`properties`下可对应有多个配置组名，如例子中的`default`；每个配置组下又有多个属性的元数据。
 
 在管理控制台的流管理页面，点击`源配置`，展开任意的源，则可展示元数据的所有配置组。
 

docs/zh_CN/plugins/functions/functions.md

@@ -1,6 +1,6 @@
 # 定制函数
 
-eKuiper 可以定制函数，函数的开发、编译及使用请[参见这里](../../extension/function.md)。
+eKuiper 可以定制函数，函数的开发、编译及使用请[参见这里](../../extension/native/function.md)。
 
 ## echo 插件
 

docs/zh_CN/sqls/overview.md

@@ -7,5 +7,5 @@ eKuiper 提供了一种类似于 SQL 的查询语言，用于对事件流执行
 - [内置函数](built-in_functions.md)
 - 扩展
     - [插件扩展](../extension/overview.md)
-    - [外部服务扩展](../extension/external_func.md)
+    - [外部服务扩展](../extension/external/external_func.md)
 

internal/plugin/portable/runtime/function.go

@@ -148,7 +148,7 @@ func (f *PortableFunc) IsAggregate() bool {
 
 func (f *PortableFunc) Close() error {
 	return f.dataCh.Close()
-	// Symbol must be cloased by instance manager
+	// Symbol must be closed by instance manager
 	//		ins.StopSymbol(ctx, c)
 }
 

internal/plugin/portable/runtime/plugin_ins_manager_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/lf-edge/ekuiper/internal/topo/state"
 	"go.nanomsg.org/mangos/v3"
 	"go.nanomsg.org/mangos/v3/protocol/req"
+	"sync"
 	"testing"
 )
 
@@ -103,6 +104,8 @@ func TestPluginInstance(t *testing.T) {
 	ctx := context.WithValue(context.Background(), context.LoggerKey, conf.Log)
 	sctx := ctx.WithMeta("rule1", "op1", &state.MemoryStore{}).WithInstance(1)
 	fmt.Printf("The test bucket size is %d.\n\n", len(tests))
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
 		err := ins.StartSymbol(sctx, tests[0].c)
 		if err != nil {
@@ -121,6 +124,7 @@ func TestPluginInstance(t *testing.T) {
 				return
 			}
 		}
+		wg.Done()
 	}()
 	// start symbol1 to avoild instance clean
 	msg, err := client.Recv()
@@ -162,6 +166,7 @@ func TestPluginInstance(t *testing.T) {
 	if err != nil {
 		t.Errorf("close ins error %v", err)
 	}
+	wg.Wait()
 }
 
 func createMockClient(pluginName string) (mangos.Socket, error) {

internal/server/rpc.go

@@ -21,6 +21,7 @@ import (
 	"github.com/lf-edge/ekuiper/internal/pkg/model"
 	"github.com/lf-edge/ekuiper/internal/plugin"
 	"github.com/lf-edge/ekuiper/internal/plugin/native"
+	"github.com/lf-edge/ekuiper/internal/plugin/portable"
 	"github.com/lf-edge/ekuiper/internal/service"
 	"github.com/lf-edge/ekuiper/internal/topo/sink"
 	"strings"
@@ -211,7 +212,11 @@ func (t *Server) CreatePlugin(arg *model.PluginDesc, reply *string) error {
 	if p.GetFile() == "" {
 		return fmt.Errorf("Create plugin error: Missing plugin file url.")
 	}
-	err = native.GetManager().Register(pt, p)
+	if pt == plugin.PORTABLE {
+		err = portable.GetManager().Register(p)
+	} else {
+		err = native.GetManager().Register(pt, p)
+	}
 	if err != nil {
 		return fmt.Errorf("Create plugin error: %s", err)
 	} else {
@@ -243,16 +248,24 @@ func (t *Server) DropPlugin(arg *model.PluginDesc, reply *string) error {
 	if err != nil {
 		return fmt.Errorf("Drop plugin error: %s", err)
 	}
-	err = native.GetManager().Delete(pt, p.GetName(), arg.Stop)
-	if err != nil {
-		return fmt.Errorf("Drop plugin error: %s", err)
+	if pt == plugin.PORTABLE {
+		err = portable.GetManager().Delete(p.GetName())
+		if err != nil {
+			return fmt.Errorf("Drop plugin error: %s", err)
+		} else {
+			*reply = fmt.Sprintf("Plugin %s is dropped .", p.GetName())
+		}
 	} else {
-		if arg.Stop {
-			*reply = fmt.Sprintf("Plugin %s is dropped and Kuiper will be stopped.", p.GetName())
+		err = native.GetManager().Delete(pt, p.GetName(), arg.Stop)
+		if err != nil {
+			return fmt.Errorf("Drop plugin error: %s", err)
 		} else {
-			*reply = fmt.Sprintf("Plugin %s is dropped and Kuiper must restart for the change to take effect.", p.GetName())
+			if arg.Stop {
+				*reply = fmt.Sprintf("Plugin %s is dropped and Kuiper will be stopped.", p.GetName())
+			} else {
+				*reply = fmt.Sprintf("Plugin %s is dropped and Kuiper must restart for the change to take effect.", p.GetName())
+			}
 		}
-
 	}
 	return nil
 }
@@ -282,7 +295,13 @@ func (t *Server) DescPlugin(arg *model.PluginDesc, reply *string) error {
 	if err != nil {
 		return fmt.Errorf("Describe plugin error: %s", err)
 	}
-	m, ok := native.GetManager().GetPluginInfo(pt, p.GetName())
+	var m interface{}
+	var ok bool
+	if pt == plugin.PORTABLE {
+		m, ok = portable.GetManager().GetPluginInfo(p.GetName())
+	} else {
+		m, ok = native.GetManager().GetPluginInfo(pt, p.GetName())
+	}
 	if !ok {
 		return fmt.Errorf("Describe plugin error: not found")
 	} else {

sdk/go/example/mirror/main.go

@@ -21,7 +21,6 @@ import (
 )
 
 func main() {
-	// TODO key case sensitive?
 	sdk.Start(os.Args, &sdk.PluginConfig{
 		Name: "mirror",
 		Sources: map[string]sdk.NewSourceFunc{