doc(*): add concepts and modify nav

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

.github/scripts/directory_check.py

@@ -27,7 +27,7 @@ def check_md_content(md_file):
     for url in url_list:
         if url[0].endswith('!'):
             continue
-        if url[2].startswith(('http://', 'https://', '<', '#')):
+        if url[2].startswith(('http://', 'https://', '<', '#', 'mailto:')) or url[2].find('.md') == -1:
             continue
         url_path = url[2].split('.md')[0]
         ref_md_path = os.path.join(f'{"/".join(md_file.split("/")[:-1])}/', f'{url_path}.md')

docs/directory.json

@@ -1,5 +1,6 @@
 {
-	"cn": [{
+	"cn": [
+		{
 			"title": "概要",
 			"path": "./"
 		},
@@ -342,184 +343,234 @@
 			]
 		}
 	],
-	"en": [{
-			"title": "Introduction",
-			"path": "./"
+	"en": [
+		{
+			"title": "Home",
+			"path": "reference"
 		},
 		{
-			"title": "Install and operation",
-			"children": [{
-					"title": "Download and install quick start",
+			"title": "Getting Started",
+			"children": [
+				{
+					"title": "Run eKuiper locally",
 					"path": "getting_started"
 				},
 				{
-					"title": "Docker installation tutorial",
+					"title": "Run eKuiper in Docker",
 					"path": "quick_start_docker"
 				},
 				{
-					"title": "EdgeX Foundry installation",
+					"title": "Run eKuiper with EdgeX Foundry",
 					"path": "edgex/edgex_rule_engine_tutorial"
-				},
-				{
-					"title": "How to use meta function to extract addtional data from EdgeX message bus",
-					"path": "edgex/edgex_meta"
 				}
 			]
 		},
 		{
-			"title": "Rules",
-			"children": [{
-					"title": "Introduction",
-					"path": "rules/overview"
+			"title": "Concepts",
+			"children": [
+				{
+					"title": "Why eKuiper",
+					"path": "concepts/ekuiper"
 				},
 				{
-					"title": "Rule Pipeline",
-					"path": "rules/rule_pipeline"
+					"title": "Rules",
+					"path": "concepts/rules"
 				},
 				{
-					"title": "Sinks",
-					"children": [{
-							"title": "EdgeX Message Bus action",
-							"path": "rules/sinks/edgex"
-						},
-						{
-							"title": "MQTT action",
-							"path": "rules/sinks/mqtt"
-						},
-						{
-							"title": "Memory action",
-							"path": "rules/sinks/memory"
-						},
+					"title": "Sources",
+					"children": [
 						{
-							"title": "Nop action",
-							"path": "rules/sinks/nop"
+							"title": "Overview",
+							"path": "concepts/sources/overview"
 						},
 						{
-							"title": "REST action",
-							"path": "rules/sinks/rest"
+							"title": "Stream",
+							"path": "concepts/sources/stream"
 						},
 						{
-							"title": "Log action",
-							"path": "rules/sinks/log"
+							"title": "Table",
+							"path": "concepts/sources/table"
 						}
 					]
 				},
 				{
-					"title": "Sources",
-					"children": [{
-							"title": "EdgeX Source",
-							"path": "rules/sources/edgex"
-						},
+					"title": "Sinks",
+					"path": "concepts/sinks"
+				},
+				{
+					"title": "SQL Queries",
+					"path": "concepts/sql"
+				},
+				{
+					"title": "Stream Processing",
+					"children": [
 						{
-							"title": "HTTP pull source",
-							"path": "rules/sources/http_pull"
+							"title": "Overview",
+							"path": "concepts/streaming/overview"
 						},
 						{
-							"title": "File source",
-							"path": "rules/sources/file"
+							"title": "Time Attribute",
+							"path": "concepts/streaming/time"
 						},
 						{
-							"title": "Memory source",
-							"path": "rules/sources/memory"
+							"title": "Windowing",
+							"path": "concepts/streaming/windowing"
 						},
 						{
-							"title": "MQTT source",
-							"path": "rules/sources/mqtt"
+							"title": "Join",
+							"path": "concepts/streaming/join"
 						}
 					]
 				},
 				{
-					"title": "Golang template",
-					"path": "rules/data_template"
-				},
-				{
-					"title": "State and Fault Tolerance",
-					"path": "rules/state_and_fault_tolerance"
+					"title": "Extensions",
+					"path": "concepts/extensions"
 				}
-
 			]
 		},
-
 		{
-			"title": "eKuiper SQL",
-			"children": [{
-					"title": "Introduction",
-					"path": "sqls/overview"
-				},
-				{
-					"title": "Streams",
-					"path": "sqls/streams"
-				},
-				{
-					"title": "Tables",
-					"path": "sqls/tables"
-				},
-				{
-					"title": "Data types",
-					"path": "sqls/data_types"
-				},
-				{
-					"title": "Query language element",
-					"path": "sqls/query_language_elements"
-				},
-				{
-					"title": "Windows",
-					"path": "sqls/windows"
-				},
+			"title": "Tutorials",
+			"children": [
 				{
-					"title": "Functions",
-					"path": "sqls/built-in_functions"
+					"title": "AI",
+					"children": [
+						{
+							"title": "Label image by tensorflow lite model with eKuiper native plugin",
+							"path": "tutorials/ai/tensorflow_lite_tutorial"
+						}
+					]
 				},
 				{
-					"title": "eKuiper lexical elements",
-					"path": "sqls/lexical_elements"
+					"title": "Working with EdgeX Foundry",
+					"children": [
+						{
+							"title": "EdgeX Foundry rule engine tutorial",
+							"path": "edgex/edgex_rule_engine_tutorial"
+						},
+						{
+							"title": "Meta function for EdgeX stream",
+							"path": "edgex/edgex_meta"
+						},
+						{
+							"title": "Command device with EdgeX eKuiper rules enginem",
+							"path": "edgex/edgex_rule_engine_command"
+						},
+						{
+							"title": "EdgeX source configuration command",
+							"path": "edgex/edgex_source_tutorial"
+						}
+					]
 				},
 				{
-					"title": "JSON Expressions",
-					"path": "sqls/json_expr"
+					"title": "Deploy by OpenYurt",
+					"path": "tutorials/deploy/openyurt_tutorial"
 				}
 			]
 		},
 		{
-			"title": "Extend eKuiper",
+			"title": "References",
 			"children": [
 				{
-					"title": "Introduction",
-					"path": "extension/overview"
+					"title": "Rules",
+					"children": [
+						{
+							"title": "Introduction",
+							"path": "rules/overview"
+						},
+						{
+							"title": "Rule Pipeline",
+							"path": "rules/rule_pipeline"
+						},
+						{
+							"title": "State and Fault Tolerance",
+							"path": "rules/state_and_fault_tolerance"
+						}
+					]
 				},
 				{
-					"title": "External function",
-					"path": "extension/external/external_func"
+					"title": "Sources",
+					"children": [
+						{
+							"title": "Overview",
+							"path": "rules/sources/overview"
+						},
+						{
+							"title": "Built-in sources",
+							"children": [
+								{
+									"title": "MQTT source",
+									"path": "rules/sources/builtin/mqtt"
+								},
+								{
+									"title": "EdgeX Source",
+									"path": "rules/sources/builtin/edgex"
+								},
+								{
+									"title": "HTTP pull source",
+									"path": "rules/sources/builtin/http_pull"
+								},
+								{
+									"title": "Memory source",
+									"path": "rules/sources/builtin/memory"
+								},
+								{
+									"title": "File source",
+									"path": "rules/sources/builtin/file"
+								}
+							]
+						},
+						{
+							"title": "Predefined source plugins",
+							"children": [
+								{
+									"title": "Zmq source",
+									"path": "rules/sources/plugin/zmq"
+								},
+								{
+									"title": "Random source",
+									"path": "rules/sources/plugin/random"
+								}
+							]
+						}
+					]
 				},
 				{
-					"title": "Native plugin develop",
+					"title": "Sinks",
 					"children": [
 						{
-							"title": "Native plugin develop overview",
-							"path": "extension/native/overview"
+							"title": "Overview",
+							"path": "rules/sinks/overview"
 						},
 						{
-							"title": "Custom native plugin develop",
+							"title": "Data Template",
+							"path": "rules/data_template"
+						},
+						{
+							"title": "Built-in sinks",
 							"children": [
 								{
-									"title": "Plugin develop introduction",
-									"path": "extension/native/develop/overview"
+									"title": "MQTT action",
+									"path": "rules/sinks/builtin/mqtt"
 								},
 								{
-									"title": "Plugin develop tutorial",
-									"path": "extension/native/develop/plugins_tutorial"
+									"title": "EdgeX Message Bus action",
+									"path": "rules/sinks/builtin/edgex"
 								},
 								{
-									"title": "Function plugin",
-									"path": "extension/native/develop/function"
+									"title": "REST action",
+									"path": "rules/sinks/builtin/rest"
 								},
 								{
-									"title": "Sink plugin",
-									"path": "extension/native/develop/sink"
+									"title": "Memory action",
+									"path": "rules/sinks/builtin/memory"
 								},
 								{
-									"title": "Source plugin",
-									"path": "extension/native/develop/source"
+									"title": "Log action",
+									"path": "rules/sinks/builtin/log"
+								},
+								{
+									"title": "Nop action",
+									"path": "rules/sinks/builtin/nop"
 								}
 							]
 						},
@@ -527,46 +578,128 @@
 							"title": "Predefined sink plugins",
 							"children": [
 								{
-									"title": "File sink",
-									"path": "extension/native/sinks/file"
+									"title": "Zmq sink",
+									"path": "rules/sinks/plugin/zmq"
 								},
 								{
-									"title": "Image sink",
-									"path": "extension/native/sinks/image"
+									"title": "File sink",
+									"path": "rules/sinks/plugin/file"
 								},
 								{
 									"title": "InfluxDB sink",
-									"path": "extension/native/sinks/influx"
+									"path": "rules/sinks/plugin/influx"
+								},
+								{
+									"title": "TDengine sink",
+									"path": "rules/sinks/plugin/tdengine"
 								},
 								{
 									"title": "Redis sink",
-									"path": "extension/native/sinks/redis"
+									"path": "rules/sinks/plugin/redis"
 								},
 								{
-									"title": "TDengine sink",
-									"path": "extension/native/sinks/tdengine"
+									"title": "Image sink",
+									"path": "rules/sinks/plugin/image"
+								}
+							]
+						}
+					]
+				},
+				{
+					"title": "SQL",
+					"children": [
+						{
+							"title": "Syntax",
+							"children": [
+								{
+									"title": "Introduction",
+									"path": "sqls/overview"
 								},
 								{
-									"title": "Zmq sink",
-									"path": "extension/native/sinks/zmq"
+									"title": "Lexical elements",
+									"path": "sqls/lexical_elements"
+								},
+								{
+									"title": "Data types",
+									"path": "sqls/data_types"
+								},
+								{
+									"title": "JSON Expressions",
+									"path": "sqls/json_expr"
+								},
+								{
+									"title": "Query language element",
+									"path": "sqls/query_language_elements"
 								}
 							]
 						},
 						{
-							"title": "Predefined source plugins",
-							"children": [{
-								"title": "Random source",
-								"path": "extension/native/sources/random"
-							},
+							"title": "Statements",
+							"children": [
 								{
-									"title": "Zmq source",
-									"path": "extension/native/sources/zmq"
+									"title": "Streams",
+									"path": "sqls/streams"
+								},
+								{
+									"title": "Tables",
+									"path": "sqls/tables"
 								}
 							]
 						},
 						{
-							"title": "Predefined sink plugins",
-							"path": "extension/native/functions/functions"
+							"title": "Windows",
+							"path": "sqls/windows"
+						},
+						{
+							"title": "Built-in Functions",
+							"path": "sqls/built-in_functions"
+						},
+						{
+							"title": "Predefined function plugins",
+							"path": "sqls/custom_functions"
+						}
+					]
+				}
+			]
+		},
+		{
+			"title": "Extension Programming",
+			"children": [
+				{
+					"title": "Introduction",
+					"path": "extension/overview"
+				},
+				{
+					"title": "Native plugin develop",
+					"children": [
+						{
+							"title": "Native plugin develop overview",
+							"path": "extension/native/overview"
+						},
+						{
+							"title": "Custom native plugin develop",
+							"children": [
+								{
+									"title": "Plugin develop introduction",
+									"path": "extension/native/develop/overview"
+								},
+								{
+									"title": "Plugin develop tutorial",
+									"path": "extension/native/develop/plugins_tutorial"
+								},
+								{
+									"title": "Function plugin",
+									"path": "extension/native/develop/function"
+								},
+								{
+									"title": "Sink plugin",
+									"path": "extension/native/develop/sink"
+								},
+								{
+									"title": "Source plugin",
+									"path": "extension/native/develop/source"
+								}
+							]
 						}
 					]
 				},
@@ -586,28 +719,73 @@
 							"path": "extension/portable/python_sdk"
 						}
 					]
+				},
+				{
+					"title": "External function",
+					"path": "extension/external/external_func"
 				}
 			]
 		},
-
 		{
-			"title": "Operation",
+			"title": "Operations",
 			"children": [
 				{
-					"title": "Install",
-					"path": "operation/install/overview"
+					"title": "Introduction",
+					"path": "operation/overview"
 				},
 				{
-					"title": "centos",
-					"path": "operation/install/cent-os"
+					"title": "Install",
+					"children": [
+						{
+							"title": "overview",
+							"path": "operation/install/overview"
+						},
+						{
+							"title": "centos",
+							"path": "operation/install/cent-os"
+						}
+					]
 				},
 				{
-					"title": "Configuration File",
-					"path": "operation/config/configuration_file"
+					"title": "Configuration",
+					"children": [
+						{
+							"title": "Configuration File",
+							"path": "operation/config/configuration_file"
+						},
+						{
+							"title": "Authentication",
+							"path": "operation/config/authentication"
+						}
+					]
 				},
 				{
-					"title": "Authentication",
-					"path": "operation/config/authentication"
+					"title": "Rest API",
+					"children": [{
+						"title": "Introduction",
+						"path": "operation/restapi/overview"
+					},
+						{
+							"title": "Streams",
+							"path": "operation/restapi/streams"
+						},
+						{
+							"title": "Tables",
+							"path": "operation/restapi/tables"
+						},
+						{
+							"title": "Rules",
+							"path": "operation/restapi/rules"
+						},
+						{
+							"title": "Plugins",
+							"path": "operation/restapi/plugins"
+						},
+						{
+							"title": "External Services",
+							"path": "operation/restapi/services"
+						}
+					]
 				},
 				{
 					"title": "Command line tool",
@@ -634,11 +812,12 @@
 					]
 				},
 				{
-					"title": "eKuiper management console",
-					"children": [{
-						"title": "Introduction",
-						"path": "operation/manager-ui/overview"
-					},
+					"title": "Management console",
+					"children": [
+						{
+							"title": "Introduction",
+							"path": "operation/manager-ui/overview"
+						},
 						{
 							"title": "How to display custom plugins in the installation list of the management console",
 							"path": "operation/manager-ui/plugins_in_manager"
@@ -646,38 +825,23 @@
 					]
 				},
 				{
-					"title": "Rest API",
-					"children": [{
-						"title": "Introduction",
-						"path": "operation/restapi/overview"
-					},
-						{
-							"title": "Streams",
-							"path": "operation/restapi/streams"
-						},
-						{
-							"title": "Tables",
-							"path": "operation/restapi/tables"
-						},
+					"title": "Compile",
+					"children": [
 						{
-							"title": "Rules",
-							"path": "operation/restapi/rules"
+							"title": "Compile",
+							"path": "operation/compile/compile"
 						},
 						{
-							"title": "Plugins",
-							"path": "operation/restapi/plugins"
-						},
-						{
-							"title": "External Services",
-							"path": "operation/restapi/services"
+							"title": "Cross Compile",
+							"path": "operation/compile/cross-compile"
 						}
 					]
-				},
-				{
-					"title": "Cross Compile",
-					"path": "operation/compile/cross-compile"
 				}
 			]
+		},
+		{
+			"title": "Contribute",
+			"path": "CONTRIBUTING"
 		}
 	]
 }

docs/CONTRIBUTING.md

@@ -43,7 +43,7 @@ $ git checkout -b <my-branch> upstream/master
 
 Take GoLand as an example, developers can debug the code: 
 
-1. Debug the whole program. Make sure all directories mentioned in [Makefile](../Makefile) build_prepare sections are created in your eKuiper root path. Add your breakpoints. Open `cmd/kuiperd/main.go`. In the main function, you'll find a green triangle in the ruler, click it and select debug. Then create your stream/rule that would run through your breakpoint, the debugger will pause there.
+1. Debug the whole program. Make sure all directories mentioned in [Makefile](../../Makefile) build_prepare sections are created in your eKuiper root path. Add your breakpoints. Open `cmd/kuiperd/main.go`. In the main function, you'll find a green triangle in the ruler, click it and select debug. Then create your stream/rule that would run through your breakpoint, the debugger will pause there.
 2. To debug a small portion of code, we recommend writing a unit test and debug it. You can go to any test file and find the same green triangle to run in debug mode. For example, `pkg/cast/cast_test.go` TestMapConvert_Funcs can run as debug.
 
 ### Testing
@@ -52,7 +52,7 @@ The eKuiper project leverages Github actions to run unit test & FVT (functional
 look at the PR status result, and make sure that all of testcases run successfully.
 
 - Write Golang unit testcases to test your code if necessary.
-- A set of [FVT testcases](../test/README.md) will be triggered with any PR submission, so please make sure that these
+- A set of [FVT testcases](../../test/README.md) will be triggered with any PR submission, so please make sure that these
   testcases can be run successfully.
 
 ### Licensing

docs/en_US/concepts/extensions.md

@@ -0,0 +1,23 @@
+# Extensions
+
+eKuiper provides built-in sources, sinks and functions as the building block for the rule. However, it is impossible to cover all external system for source/sink connection such as user's system with private protocol. Moreover, the built-in function cannot cover all the computation needed for all users. Thus, customized source, sink and functions are needed in many cases. eKuiper provide extension mechanism for users to customize all these three aspects.
+
+## Extension Points
+
+We support 3 extension points:
+
+- Source: add new source type for eKuiper to consume data from. The new extended source can be used in the stream/table definition.
+- Sink: add new sink type for eKuiper to produce data to. The new extended sink can be used in the rule actions definition.
+- Function: add new function type for eKuiper to process data. The new extended function can be used in the rule SQL.
+
+## Extension Types
+
+We support 3 kinds of extension:
+
+- [Go native plugin](../extension/native/overview.md): extend as a Go plugin. It is the most performant, but has a lot of limitation in development and deployment.
+- [Portable plugin](../extension/portable/overview.md) with Go or Python language, and it will support more languages later. It simplifies the development and deployment and has less limitations.
+- [External service](../extension/external/external_func.md): wrap existing external REST or rpc services as a eKuiper SQL function by configurations. It is a speedy way to extend by existing services. But it only supports function extension.
+
+## More Readings
+
+- [Extension Reference](../extension/overview.md)

docs/en_US/concepts/rules.md

@@ -0,0 +1,19 @@
+# Rules
+
+Each rule represents a computing job to run in eKuiper. It defines the continuous streaming data source as the input, the computing logic and the result actions as the output.
+
+## Rule Lifecycle
+
+Currently, eKuiper only supports stream processing rule, which means that at lease one of the rule source must be a continuous stream. Thus, the rule will run continuously once started and only stopped if the user send stop command explicitly. The rule may stop abnormally for errors or the eKuiper instance exits.
+
+## Rules Relationship
+
+It is common to run multiple rules simultaneously. As eKuiper is a single instance process, the rules are running in the same memory space. However, there are separated in the runtime and the error in one rule should not affect others. Regarding workload, all rules share the same hardware resource. Each rule can specify the operator buffer to limit the processing rate to avoid taking all resources.
+
+## Rule Pipeline
+
+Multiple rules can form a processing pipeline by specifying a joint point in sink/source. For example, the first rule produce the result to a topic in memory sink and the other rule subscribe to that topic in its memory source. Besides the pair of memory sink/source, users can also use mqtt or other sink/source pair to connect rules.
+
+## More Readings
+
+- [Rule Reference](../rules/overview.md)

docs/en_US/concepts/sinks.md

@@ -0,0 +1,13 @@
+# Sinks
+
+Sinks are used to write data to an external system. Sinks can be used to write control data to trigger an action. Sinks can also be used to write status data and save in an external storage.
+
+In a rule, the sink type are used as an action. A rule can have more than one actions and the differenct actions can be the same sink type.
+
+## Result Encoding
+
+The sink result is a string as always. It will be encoded into json string by default. Users can change the format by setting `dataTemplate` which leverage the go template syntax to format the result into a string. For even detail control of the result format, users can develop a sink extension.
+
+## More Readings
+
+- [Sink Reference](../rules/sinks/overview.md)

docs/en_US/concepts/sources/overview.md

@@ -0,0 +1,30 @@
+# Sources
+
+Sources are used to read data from external systems. The source can be unbounded streaming data named stream or bounded batch data named table. When using in a rule, at least one of  the source must be a stream.
+
+The source basically defines how to connect to an external resource and fetch data from the resource in a streaming way. After fetching the data, common tasks like decode and transform by schema can be done by setting properties.
+
+## Define and Run
+
+When define a source stream or table, it actually creates the logical definition instead of a physical running data input. The logical definition can then be used in rule's SQL in the `from` clause. The source only starts to run when any of the rules refer to it has started.
+
+By default, if multiple rules refer to the same source. Each rule will have its own, standalone source instance from other rules so that the rules are total separated. To boost performance when users want to process the same data across multiple rules, they can define the source as shared. Then the rules refer to the same shared source will share the same running source instance.
+
+## Decode
+
+Users can define the format to decode by setting `format` property. Currently, only `json` and `binary` format are supported. For other formats, customized source must be developed.
+
+## Schema
+
+Users can define the data schema like a common SQL table. In eKuiper runtime, the data will be validated and transformed according to the schema. To avoid conversion overhead if the data is fixed and clean or to consume unknown schema data, users can define schemaless source.
+
+## Stream & Table
+
+The source defines the external system connection. When using in a rule, users can define them as stream or table according to the processing mechanism. Check [stream](stream.md) and [table](table.md) for detail.
+
+## More Readings
+
+- [Source Reference](../../rules/sources/overview.md)
+
+
+

docs/en_US/concepts/sources/stream.md

@@ -0,0 +1,10 @@
+# Stream
+
+A stream is the runtime form of a source in eKuiper. It must specify a source type to define how to connect to the external resource.
+
+When using as a stream, the source must be unbounded. The stream acts like a trigger for the rule. Each event will trigger a calculation in the rule.
+
+## More Readings
+
+- [Stream Reference](../../sqls/streams.md)
+

docs/en_US/concepts/sources/table.md

@@ -0,0 +1,17 @@
+# Table
+
+In eKuiper, a table is a snapshot of the source data. In contrast to the common static tables that represent batch data, eKuiper tables can change over time.
+
+The source for table can be either bounded or unbounded. For bounded source table, the content of the table is static. For unbounded table, the content of the table is dynamic.
+
+## Table Updates
+
+Currently, the table update in eKuiper is append-only. Users can specify the properties to limit the table size to avoid too much memory consumption.
+
+## Table Usages
+
+Table cannot be used standalone in a rule. It is usually used to join with streams. It can be used to enrich stream data or as a switch for calculation.
+
+## More Readings
+
+- [Table Reference](../../sqls/tables.md)

docs/en_US/concepts/sql.md

@@ -0,0 +1,11 @@
+# SQL
+
+The SQL language support in eKuiper includes Data Definition Language (DDL), Data Manipulation Language (DML) and Query Language. The SQL support in eKuiper is a subset of ANSI SQL and has some customized extensions.
+
+## SQL in source definition
+
+When create and manage stream or table source, SQL DDL and DML are used as the command payload. Check [streams](../sqls/streams.md) and [tables](../sqls/tables.md) for detail.
+
+## SQL queries in rules
+
+In rules, SQL queries are used to define the business logic. Please check [sql reference](../sqls/overview.md) for detail.

docs/en_US/concepts/streaming/join.md

@@ -0,0 +1,10 @@
+# Join of sources
+
+Currently, join is the only way to merge multiple sources in eKuiper. It requires a way to align multiple sources and trigger the join result.
+
+The supported joins in eKuiper include:
+
+- Join of streams: must do in a window.
+- Join of stream and table: the stream will be the trigger of join operation.
+
+The supported join type includes LEFT, RIGHT, FULL & CROSS in eKuiper.

docs/en_US/concepts/streaming/windowing.md

@@ -0,0 +1,12 @@
+# Windowing
+
+As streaming data is infinite, it is impossible to process it as a whole. Windowing provides a mechanism to split the unbounded data into a continuous series of bounded data to calculate.
+
+In eKuiper, the built-in windowing supports:
+
+- Time window: window split by time
+- Count window: window split by element count
+
+In time window, both processing time and event time are supported.
+
+For all the supported window type, please check [window functions](../../sqls/windows.md).

docs/en_US/edgex/edgex_rule_engine_tutorial.md

@@ -32,7 +32,7 @@ EdgeX uses [message bus](https://github.com/edgexfoundry/go-mod-messaging) to ex
   CREATE STREAM demo (temperature bigint) WITH (FORMAT="JSON"...)
   ```
 
-  However, data type definitions are already specified in the EdgeX events/readings and to improve the using experience, user are NOT necessary to specify data types when creating stream. For any data sending from message bus, it will be converted into [corresponding data types](../rules/sources/edgex.md).
+  However, data type definitions are already specified in the EdgeX events/readings and to improve the using experience, user are NOT necessary to specify data types when creating stream. For any data sending from message bus, it will be converted into [corresponding data types](../rules/sources/builtin/edgex.md).
 
 - An EdgeX message bus sink is extended to support send analysis result back to EdgeX Message Bus. User can also choose to send analysis result to RestAPI, eKuiper already supported it. 
 
@@ -94,7 +94,7 @@ add these in `environment` part and make sure the image is `1.4.0` or later.
   ```
   
 * `mqtt/zeromq` messageBus: adjust the parameters accordingly and specify the client credentials if have.
-  There is a `mqtt` message bus example, make sure the connection info exists in `etc/connections/connection.yaml`, for [more info](../rules/sources/edgex.md#connectionselector) please check this. 
+  There is a `mqtt` message bus example, make sure the connection info exists in `etc/connections/connection.yaml`, for [more info](../rules/sources/builtin/edgex.md#connectionselector) please check this. 
   ```yaml
   environment:
       CONNECTION__EDGEX__MQTTMSGBUS__PORT: 1883
@@ -105,7 +105,7 @@ add these in `environment` part and make sure the image is `1.4.0` or later.
       CONNECTION__EDGEX__MQTTMSGBUS__OPTIONAL__PASSWORD: password
       EDGEX__DEFAULT__CONNECTIONSELECTOR: edgex.mqttMsgBus
   ```
-After these modifications and eKuiper starts up, please read [this](../rules/sinks/edgex.md#connection-reuse-publish-example) to learn how to refer to the connection info
+After these modifications and eKuiper starts up, please read [this](../rules/sinks/builtin/edgex.md#connection-reuse-publish-example) to learn how to refer to the connection info
 
 #### Use Redis as KV storage
 
@@ -183,11 +183,11 @@ default:
 .....  
 ```
 
-For more detailed information of configuration file, please refer to [this doc](../rules/sources/edgex.md).
+For more detailed information of configuration file, please refer to [this doc](../rules/sources/builtin/edgex.md).
 
 ### Create a rule
 
-Let's create a rule that send result data to an MQTT broker, for detailed information of MQTT sink, please refer to [this link](../rules/sinks/mqtt.md).  Similar to create a stream, you can also choose REST or CLI to manage rules. 
+Let's create a rule that send result data to an MQTT broker, for detailed information of MQTT sink, please refer to [this link](../rules/sinks/builtin/mqtt.md).  Similar to create a stream, you can also choose REST or CLI to manage rules. 
 
 So the below rule will get all of values from `event` topic. The sink result will 
 
@@ -340,10 +340,10 @@ Current rule does not filter any data that are sent to eKuiper, so how to filter
 #### Extended Reading
 
 - Starting from eKuiper 0.9.1 version, [a visualized web UI](../operation/manager-ui/overview.md) is released with a separated Docker image. You can manage the streams, rules and plugins through web page. 
-- Read [EdgeX source](../rules/sources/edgex.md) for more detailed information of configurations and data type conversion.
+- Read [EdgeX source](../rules/sources/builtin/edgex.md) for more detailed information of configurations and data type conversion.
 - [How to use meta function to extract additional data from EdgeX message bus?](edgex_meta.md) There are some other information are sent along with device service, such as event created time, event id etc. If you want to use such metadata information in your SQL statements, please refer to this doc.
 - [Use Golang template to customize analaysis result in eKuiper](../rules/data_template.md) Before the analysis result is sent to different sinks, the data template can be used to make more processing. You can refer to this doc for more scenarios of using data templates.
-- [EdgeX message bus sink doc](../rules/sinks/edgex.md). The document describes how to use EdgeX message bus sink. If you'd like to have your analysis result be consumed by other EdgeX services, you can send analysis data with EdgeX data format through this sink, and other EdgeX services can subscribe new message bus exposed by eKuiper sink.
+- [EdgeX message bus sink doc](../rules/sinks/builtin/edgex.md). The document describes how to use EdgeX message bus sink. If you'd like to have your analysis result be consumed by other EdgeX services, you can send analysis data with EdgeX data format through this sink, and other EdgeX services can subscribe new message bus exposed by eKuiper sink.
 - [eKuiper plugin development tutorial](../extension/native/develop/plugins_tutorial.md): eKuiper plugin is based on the plugin mechanism of Golang, users can build loosely-coupled plugin applications,  dynamic loading and binding when it is running. You can refer to this article if you're interested in eKuiper plugin development.
 
  If you want to explore more features of eKuiper, please refer to below resources.

docs/en_US/edgex/edgex_source_tutorial.md

@@ -1,6 +1,6 @@
 # Configure the data flow from EdgeX to eKuiper
 
-Sources feed data into eKuiper from other systems such as EdgeX foundry which are defined as streams. [EdgeX source](../rules/sources/edgex.md) defines the properties to configure how the data feed into eKuiper from EdgeX. In this tutorial, we will demonstrate the various data flow from EdgeX to eKuiper and how to configure the source to adopt any kind of data flow.
+Sources feed data into eKuiper from other systems such as EdgeX foundry which are defined as streams. [EdgeX source](../rules/sources/builtin/edgex.md) defines the properties to configure how the data feed into eKuiper from EdgeX. In this tutorial, we will demonstrate the various data flow from EdgeX to eKuiper and how to configure the source to adopt any kind of data flow.
 
 ## Typical Data Flow Model
 
@@ -15,7 +15,7 @@ Notice that, the EdgeX message bus receives messages from various service such a
 
 By default, the first kind of data flow is used which allow users to prepare (transformed, enriched, filtered, etc.) and groom (formatted, compressed, encrypted, etc.) before sending to the eKuiper rule engine. If users don't need to transform the data and would like to process the raw data in eKuiper to reduce the overhead, they can connect to the message bus directly.
 
-The full properties list of EdgeX source can be found [here](../rules/sources/edgex.md#global-configurations). There are two critical properties that define the connection model: `topic` and `messageType`. Let's explore how to configure them to adopt the connection models.
+The full properties list of EdgeX source can be found [here](../rules/sources/builtin/edgex.md#global-configurations). There are two critical properties that define the connection model: `topic` and `messageType`. Let's explore how to configure them to adopt the connection models.
 
 ## Connect to the App Service
 

docs/en_US/extension/native/develop/overview.md

@@ -10,10 +10,10 @@ Developers of eKuiper plugin can specify metadata files during the development p
 
 ## Sources
 
-| Name                        | Descriptiom                                                  | Remarks                                                   |
-| --------------------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
-| [zmq](../sources/zmq.md)       | The plugin listens to Zero Mq messages and sends them to the eKuiper stream | Sample of plugin, not available in production environment |
-| [random](../sources/random.md) | The plugin generates messages according to the specified pattern | Sample of plugin, not available in production environment |
+| Name                                              | Description                                                                 | Remarks                                                   |
+|---------------------------------------------------|-----------------------------------------------------------------------------|-----------------------------------------------------------|
+| [zmq](../../../rules/sources/plugin/zmq.md)       | The plugin listens to Zero Mq messages and sends them to the eKuiper stream | Sample of plugin, not available in production environment |
+| [random](../../../rules/sources/plugin/random.md) | The plugin generates messages according to the specified pattern            | Sample of plugin, not available in production environment |
 
 ### source metadata file format
 
@@ -157,12 +157,12 @@ The following is a sample of metadata file.
 
 ## Sinks/Actions
 
-| Name                        | Description                                                  | Remarks                                                   |
-| --------------------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
-| [file](../sinks/file.md)       | The plugin saves the analysis results to a specified file system | Sample of plugin, not available in production environment |
-| [zmq](../sinks/zmq.md)         | The plugin sends the analysis results to the topic of Zero Mq | Sample of plugin, not available in production environment |
-| [Influxdb](../sinks/influx.md) | The plugin sends the analysis results to InfluxDB            | Provided by [@smart33690](https://github.com/smart33690)  |
-| [TDengine](../sinks/tdengine.md)   | The plugin sends the analysis results to TDengine            |                                                           |
+| Name                                                | Description                                                      | Remarks                                                   |
+|-----------------------------------------------------|------------------------------------------------------------------|-----------------------------------------------------------|
+| [file](../../../rules/sinks/plugin/file.md)         | The plugin saves the analysis results to a specified file system | Sample of plugin, not available in production environment |
+| [zmq](../../../rules/sinks/plugin/zmq.md)           | The plugin sends the analysis results to the topic of Zero Mq    | Sample of plugin, not available in production environment |
+| [Influxdb](../../../rules/sinks/plugin/influx.md)   | The plugin sends the analysis results to InfluxDB                | Provided by [@smart33690](https://github.com/smart33690)  |
+| [TDengine](../../../rules/sinks/plugin/tdengine.md) | The plugin sends the analysis results to TDengine                |                                                           |
 
 ### sink metadata file format
 
@@ -276,13 +276,13 @@ The following is a sample of metadata file.
 
 ## Functions
 
-| Name                                          | Description                                                  | Remarks                                                 |
-| --------------------------------------------- | ------------------------------------------------------------ | ------------------------------------------------------- |
-| [echo](../functions/functions.md)                | Output parameter value as it is                              | Plugin sample, not available for production environment |
-| [countPlusOne](../functions/functions.md)        | Output the value of the parameter length plus one            | Plugin sample, not available for production environment |
-| [accumulateWordCount](../functions/functions.md) | The function counts how many words there are                 | Plugin sample, not available for production environment |
-| [resize](../functions/functions.md)              | Create a scaled image with new dimensions (width, height). If width or height is set to 0, it is set to the reserved value of aspect ratio | Plugin sample, not available for production environment |
-| [thumbnail](../functions/functions.md)           | Reduce the image that retains the aspect ratio to the maximum size (maxWidth, maxHeight). | Plugin sample, not available for production environment |
+| Name                                                     | Description                                                                                                                                | Remarks                                                 |
+|----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
+| [echo](../../../sqls/custom_functions.md)                | Output parameter value as it is                                                                                                            | Plugin sample, not available for production environment |
+| [countPlusOne](../../../sqls/custom_functions.md)        | Output the value of the parameter length plus one                                                                                          | Plugin sample, not available for production environment |
+| [accumulateWordCount](../../../sqls/custom_functions.md) | The function counts how many words there are                                                                                               | Plugin sample, not available for production environment |
+| [resize](../../../sqls/custom_functions.md)              | Create a scaled image with new dimensions (width, height). If width or height is set to 0, it is set to the reserved value of aspect ratio | Plugin sample, not available for production environment |
+| [thumbnail](../../../sqls/custom_functions.md)           | Reduce the image that retains the aspect ratio to the maximum size (maxWidth, maxHeight).                                                  | Plugin sample, not available for production environment |
 
 eKuiper has many built-in functions that can perform calculations on data. (Refer to https://github.com/lf-edge/ekuiper/blob/master/docs/zh_CN/sqls/built-in_functions.md for specific documentation)
 

docs/en_US/extension/native/develop/sink.md

@@ -1,6 +1,6 @@
 # Sink Extension
 
-Sink feed data from eKuiper into external systems. eKuiper has built-in sink support for [MQTT broker](../../../rules/sinks/mqtt.md) and [log sink](../../../rules/sinks/log.md). There are still needs to publish data to various external systems include messaging systems and database etc. Sink extension is presented to meet this requirement.
+Sink feed data from eKuiper into external systems. eKuiper has built-in sink support for [MQTT broker](../../../rules/sinks/builtin/mqtt.md) and [log sink](../../../rules/sinks/builtin/log.md). There are still needs to publish data to various external systems include messaging systems and database etc. Sink extension is presented to meet this requirement.
 
 ## Developing
 
@@ -8,9 +8,9 @@ Sink feed data from eKuiper into external systems. eKuiper has built-in sink sup
 
 To develop a sink for eKuiper is to implement [api.Sink](https://github.com/lf-edge/ekuiper/blob/master/pkg/api/stream.go) interface and export it as a golang plugin.
 
-Before starting the development, you must [setup the environment for golang plugin](../../overview.md#setup-the-plugin-developing-environment). 
+Before starting the development, you must [setup the environment for golang plugin](../overview.md#setup-the-plugin-developing-environment). 
 
-To develop a sink, the _Configure_ method must be implemented. This method will be called once the sink is initialized. In this method, a map that contains the configuration in the [rule actions definition](../../../rules/overview.md#actions) is passed in. Typically, there will be information such as host, port, user and password of the external system. You can use this map to initialize this sink.
+To develop a sink, the _Configure_ method must be implemented. This method will be called once the sink is initialized. In this method, a map that contains the configuration in the [rule actions definition](../../../rules/overview.md#sinksactions) is passed in. Typically, there will be information such as host, port, user and password of the external system. You can use this map to initialize this sink.
 
 ```go
 //Called during initialization. Configure the sink with the properties from action definition 
@@ -43,7 +43,7 @@ The last method to implement is _Close_ which literally close the connection. It
 Close(ctx StreamContext) error
 ```
 
-As the sink itself is a plugin, it must be in the main package. Given the sink struct name is mySink. At last of the file, the sink must be exported as a symbol as below. There are [2 types of exported symbol supported](../../overview.md#plugin-development). For sink extension, states are usually needed, so it is recommended to export a constructor function.
+As the sink itself is a plugin, it must be in the main package. Given the sink struct name is mySink. At last of the file, the sink must be exported as a symbol as below. There are [2 types of exported symbol supported](../overview.md#plugin-development). For sink extension, states are usually needed, so it is recommended to export a constructor function.
 
 ```go
 func MySink() api.Sink {
@@ -74,7 +74,7 @@ go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/sink
 
 ### Usage
 
-The customized sink is specified in a [actions definition](../../../rules/overview.md#actions). Its name is used as the key of the action. The configuration is the value.
+The customized sink is specified in a [actions definition](../../../rules/overview.md#sinksactions). Its name is used as the key of the action. The configuration is the value.
 
 If you have developed a sink implementation MySink, you should have:
 1. In the plugin file, symbol MySink is exported.

docs/en_US/extension/native/overview.md

@@ -2,8 +2,8 @@
 
 eKuiper allows user to customize the different kinds of extensions by the native golang plugin system. 
 
-- The source extension is used for extending different stream source, such as consuming data from other message brokers. eKuiper has built-in source support for [MQTT broker](../../rules/sources/mqtt.md).
-- Sink/Action extension is used for extending pub/push data to different targets, such as database, other message system, web interfaces or file systems. Built-in action is supported in eKuiper, see [MQTT](../../rules/sinks/mqtt.md) & [log files](../../rules/sinks/log.md).
+- The source extension is used for extending different stream source, such as consuming data from other message brokers. eKuiper has built-in source support for [MQTT broker](../../rules/sources/builtin/mqtt.md).
+- Sink/Action extension is used for extending pub/push data to different targets, such as database, other message system, web interfaces or file systems. Built-in action is supported in eKuiper, see [MQTT](../../rules/sinks/builtin/mqtt.md) & [log files](../../rules/sinks/builtin/log.md).
 - Functions extension allows user to extend different functions that used in SQL. Built-in functions is supported in eKuiper, see [functions](../../sqls/built-in_functions.md).
 
 Please read the following to learn how to implement different extensions.
@@ -19,6 +19,37 @@ We recommend plugin name to be camel case. Notice that, there are some restricti
 1. The name of the export symbol of the plugin should be camel case with an **upper case first letter**. It must be the same as the plugin name except the first letter. For example, plugin name _file_ must export a export symbol name _File_ .
 2. The name of _.so_ file must be the same as the export symbol name or the plugin name. For example, _MySource.so_ or _mySink.so_.
 
+### Version
+
+The user can **optionally** add a version string to the name of _.so_ to help identify the version of the plugin. The version can be then retrieved through describe CLI command or REST API. The naming convention is to add a version string to the name after _@_. The version can be any string. If the version string starts with "v", the "v" will be ignored in the return result. Below are some typical examples.
+
+- _MySource@v1.0.0.so_ : version is 1.0.0
+- _MySource@20200331.so_:  version is 20200331
+
+If multiple versions of plugins with the same name in place, only the latest version(ordered by the version string) will be taken effect.
+
+## Setup the plugin developing environment
+
+It is required to build the plugin with exactly the same version of dependencies. And the plugin must implement interfaces exported by Kuiper, so the Kuiper project must be in the gopath.
+
+A typical environment for developing plugins is to put the plugin and Kuiper in the same project. To set it up:
+1. Clone Kuiper project.
+2. Create the plugin implementation file inside plugins/sources or plugin/sinks or plugin/functions according to what extension type is developing.
+3. Build the file as plugin into the same folder. The build command is typically like:
+```bash
+go build -trimpath --buildmode=plugin -o plugins/sources/MySource.so plugins/sources/my_source.go
+```
+
+Notice that, the `-trimpath` build flag is required if using the prebuilte kuiper or kuiper docker image because the kuiperd is also built with the flag to improve build reproducibility.
+
+### Plugin development
+
+The development of plugins is to implement a specific interface according to the plugin type and export the implementation with a specific name. There are two types of exported symbol supported:
+
+1. Export a constructor function: Kuiper will use the constructor function to create a new instance of the plugin implementation for each load. So each rule will have one instance of the plugin and each instance will be isolated from others. This is the recommended way.
+
+2. Export an instance: Kuiper will use the instance as singleton for all plugin load. So all rules will share the same instance. For such implementation, the developer will need to handle the shared states to avoid any potential multi-thread problems. This mode is recommended where there are no shared states and the performance is critical. Especially, function extension is usually functional without internal state which is suitable for this mode.
+
 ## State storage
 
 eKuiper extension exposes a key value state storage interface through the context parameter, which can be used for all types of extensions, including Source/Sink/Function extensions.

docs/en_US/operation/config/configuration_file.md

@@ -94,11 +94,11 @@ The URL where hosts all of pre-build [native plugins](../../extension/native/ove
 
 All plugins list as follows:
 
-|  plugin types  |        pre-build plugins                                        |
-|  -----------   |  ---------------------------------------------------------------|
-|  source        |    random zmq                                                   |
-|  sink          |  file image influx redis tdengine zmq                           |
-|  function      |  accumulateWordCount countPlusOne echo geohash image labelImage |
+| plugin types | pre-build plugins                                              |
+|--------------|----------------------------------------------------------------|
+| source       | random zmq                                                     |
+| sink         | file image influx redis tdengine zmq                           |
+| function     | accumulateWordCount countPlusOne echo geohash image labelImage |
 
 User can get all pre-build plugins names and address by below Rest-APIs:
 
@@ -146,7 +146,7 @@ It has properties
 * connectionSelector - reuse the connection info defined in etc/connections/connection.yaml, mainly used for edgeX redis in secure mode
   * only applicable to redis connection information
   * the server, port and password in connection info will overwrite the host port and password above
-  * [more info](../../rules/sources/edgex.md#connectionselector)
+  * [more info](../../rules/sources/builtin/edgex.md#connectionselector)
     
 
 ### Config

docs/en_US/operation/manager-ui/overview.md

@@ -144,7 +144,7 @@ As shown in the figure below, a rule named demoRule is created to filter out the
 
 ![newRule](./resources/new_rule.png)
 
-Click the "Add" button and a dialog box will pop up as shown below. The file path of `/ekuiper/demoFile` where the input result is stored is input . More information about the file sink can be found in [Help File](../../extension/native/sinks/file.md). The target file is in the `Beta` state and cannot be used as an actual production environment.
+Click the "Add" button and a dialog box will pop up as shown below. The file path of `/ekuiper/demoFile` where the input result is stored is input . More information about the file sink can be found in [Help File](../../rules/sinks/plugin/file.md). The target file is in the `Beta` state and cannot be used as an actual production environment.
 
 ![sinkConf](./resources/sink_conf.png)
 

docs/en_US/operation/overview.md

@@ -1,10 +1,10 @@
 
 eKuiper is developed by Golang, and it can be run at different operating systems. See below docs for how to install and operating eKuiper.
 
-- [Compile instruction](compile/compile.md)
-- [Install instruction](install/overview.md)
-- [Configuration File](config/configuration_file.md)
-- [Cli management](cli/overview.md)
-- [UI management](manager-ui/overview.md)
-- [rest api management](restapi/overview.md)
+- [Compile instruction](./compile/compile.md)
+- [Install instruction](./install/overview.md)
+- [Configuration File](./config/configuration_file.md)
+- [Cli management](./cli/overview.md)
+- [UI management](./manager-ui/overview.md)
+- [rest api management](./restapi/overview.md)
 

docs/en_US/rules/overview.md

@@ -61,9 +61,9 @@ The rule options can be defined globally in `etc/kuiper.yaml` under the `rules`
 ## Sources
 
 - eKuiper provides embeded following 3 sources,
-  - MQTT source, see  [MQTT source stream](./sources/mqtt.md) for more detailed info.
-  - EdgeX source by default is shipped in [docker images](https://hub.docker.com/r/lfedge/ekuiper), but NOT included in single download binary files, you use `make pkg_with_edgex` command to build a binary package that supports EdgeX source. Please see [EdgeX source stream](./sources/edgex.md) for more detailed info.
-  - HTTP pull source, regularly pull the contents at user's specified interval time, see [here](./sources/http_pull.md) for more detailed info.
+  - MQTT source, see  [MQTT source stream](./sources/builtin/mqtt.md) for more detailed info.
+  - EdgeX source by default is shipped in [docker images](https://hub.docker.com/r/lfedge/ekuiper), but NOT included in single download binary files, you use `make pkg_with_edgex` command to build a binary package that supports EdgeX source. Please see [EdgeX source stream](./sources/builtin/edgex.md) for more detailed info.
+  - HTTP pull source, regularly pull the contents at user's specified interval time, see [here](./sources/builtin/http_pull.md) for more detailed info.
 - See [SQL](../sqls/overview.md) for more info of eKuiper SQL.
 - Sources can be customized, see [extension](../extension/overview.md) for more detailed info.
 
@@ -71,11 +71,11 @@ The rule options can be defined globally in `etc/kuiper.yaml` under the `rules`
 
 Currently, below kinds of sinks/actions are supported:
 
-- [log](./sinks/log.md): Send the result to log file.
-- [mqtt](./sinks/mqtt.md): Send the result to an MQTT broker.
-- [edgex](./sinks/edgex.md): Send the result to EdgeX message bus.
-- [rest](./sinks/rest.md): Send the result to a Rest HTTP server.
-- [nop](./sinks/nop.md): Send the result to a nop operation.
+- [log](./sinks/builtin/log.md): Send the result to log file.
+- [mqtt](./sinks/builtin/mqtt.md): Send the result to an MQTT broker.
+- [edgex](./sinks/builtin/edgex.md): Send the result to EdgeX message bus.
+- [rest](./sinks/builtin/rest.md): Send the result to a Rest HTTP server.
+- [nop](./sinks/builtin/nop.md): Send the result to a nop operation.
 
 Each action can define its own properties. There are several common properties:
 

docs/en_US/rules/rule_pipeline.md

@@ -1,6 +1,6 @@
 # Rule Pipeline
 
-We can form rule pipelines by importing results of prior rule into the following rule. This is possible by employing intermediate storage or MQ such as mqtt broker. By using the pair of [memory source](./sources/memory.md) and [sink](./sinks/memory.md), we can create rule pipelines without external dependencies.
+We can form rule pipelines by importing results of prior rule into the following rule. This is possible by employing intermediate storage or MQ such as mqtt broker. By using the pair of [memory source](./sources/builtin/memory.md) and [sink](./sinks/builtin/memory.md), we can create rule pipelines without external dependencies.
 
 ## Usage
 

docs/en_US/rules/sinks/edgex.md

@@ -6,21 +6,21 @@ The action is used for publishing output message into EdgeX message bus.
 
 **Also, you need to expose the port number to host server before running the eKuiper server if you want to have the service available to other hosts.**
 
-| Property name      | Optional | Description                                                                                                                                                                                                                                                                                  |
-|--------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| type               | true     | The message bus type, three types of message buses are supported, `zero`, `mqtt` and `redis`, and `redis` is the default value.                                                                                                                                                      |
+| Property name      | Optional | Description                                                                                                                                                                                                                                                                                |
+|--------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| type               | true     | The message bus type, three types of message buses are supported, `zero`, `mqtt` and `redis`, and `redis` is the default value.                                                                                                                                                            |
 | protocol           | true     | The protocol. If it's not specified, then use default value `redis`.                                                                                                                                                                                                                       |
 | host               | true     | The host of message bus. If not specified, then use default value `localhost`.                                                                                                                                                                                                             |
 | port               | true     | The port of message bus. If not specified, then use default value `6379`.                                                                                                                                                                                                                  |
-| connectionSelector | true     | reuse the connection to EdgeX message bus. [more info](../sources/edgex.md#connectionselector)                                                                                                                                                                                               |
+| connectionSelector | true     | reuse the connection to EdgeX message bus. [more info](../../sources/builtin/edgex.md#connectionselector)                                                                                                                                                                                  |
 | topic              | true     | The topic to be published. The topic is static across all messages. To use dynamic topic, leave this empty and specify the topicPrefix property. Only one of the topic and topicPrefix properties can be specified. If both are not specified, then use default topic value `application`. |
-| topicPrefix        | true     | The prefix of a dynamic topic to be published. The topic will become a concatenation of `$topicPrefix/$profileName/$deviceName/$sourceName`.                                                                                                                                                 |
+| topicPrefix        | true     | The prefix of a dynamic topic to be published. The topic will become a concatenation of `$topicPrefix/$profileName/$deviceName/$sourceName`.                                                                                                                                               |
 | contentType        | true     | The content type of message to be published. If not specified, then use the default value `application/json`.                                                                                                                                                                              |
 | messageType        | true     | The EdgeX message model type. To publish the message as an event like EdgeX application service, use `event`. Otherwise, to publish the message as an event request like EdgeX device service or core data service, use `request`. If not specified, then use the default value `event`.   |
 | metadata           | true     | The property is a field name that allows user to specify a field name of SQL  select clause,  the field name should use `meta(*) AS xxx`  to select all of EdgeX metadata from message.                                                                                                    |
-| profileName        | true     | Allows user to specify the profile name in the event structure that are sent from eKuiper. The profileName in the meta take precedence if specified.                                                                                                                                         |
-| deviceName         | true     | Allows user to specify the device name in the event structure that are sent from eKuiper. The deviceName in the meta take precedence if specified.                                                                                                                                           |
-| sourceName         | true     | Allows user to specify the source name in the event structure that are sent from eKuiper. The sourceName in the meta take precedence if specified.                                                                                                                                           |
+| profileName        | true     | Allows user to specify the profile name in the event structure that are sent from eKuiper. The profileName in the meta take precedence if specified.                                                                                                                                       |
+| deviceName         | true     | Allows user to specify the device name in the event structure that are sent from eKuiper. The deviceName in the meta take precedence if specified.                                                                                                                                         |
+| sourceName         | true     | Allows user to specify the source name in the event structure that are sent from eKuiper. The sourceName in the meta take precedence if specified.                                                                                                                                         |
 | optional           | true     | If `mqtt` message bus type is specified, then some optional values can be specified. Please refer to below for supported optional supported configurations.                                                                                                                                |
 
 Below optional configurations are supported, please check MQTT specification for the detailed information.
@@ -145,7 +145,7 @@ Below is a rule that send analysis result to zeromq message bus.
 ## Connection reuse publish example
 
 Below is an example for how to use connection reuse feature. We just need remove the connection related parameters and
-use the `connectionSelector` to specify the connection to reuse. [more info](../sources/edgex.md#connectionselector)
+use the `connectionSelector` to specify the connection to reuse. [more info](../../sources/builtin/edgex.md#connectionselector)
 
 ```json
 {

docs/en_US/rules/sinks/memory.md

@@ -1,6 +1,6 @@
 # Memory action
 
-The action is used to flush the result into an in-memory topic so that it can be consumed by the [memory source](../sources/memory.md). The topic is like pubsub topic such as mqtt, so that there could be multiple memory sinks which publish to the same topic and multiple memory sources which subscribe to the same topic. The typical usage for memory action is to form [rule pipelines](../rule_pipeline.md).
+The action is used to flush the result into an in-memory topic so that it can be consumed by the [memory source](../../sources/builtin/memory.md). The topic is like pubsub topic such as mqtt, so that there could be multiple memory sinks which publish to the same topic and multiple memory sources which subscribe to the same topic. The typical usage for memory action is to form [rule pipelines](../../rule_pipeline.md).
 
 | Property name | Optional | Description                                    |
 |---------------|----------|------------------------------------------------|

docs/en_US/rules/sinks/mqtt.md

@@ -16,7 +16,7 @@ The action is used for publish output message into an MQTT server.
 | rootCaPath         | true     | The location of root ca path. It can be an absolute path, or a relative path, which is similar to use of certificationPath.                                                                                                                                                                                                                               |
 | insecureSkipVerify | true     | If InsecureSkipVerify is `true`, TLS accepts any certificate presented by the server and any host name in that certificate.  In this mode, TLS is susceptible to man-in-the-middle attacks. The default value is `false`. The configuration item can only be used with TLS connections.                                                                   |
 | retained           | true     | If retained is `true`,The broker stores the last retained message and the corresponding QoS for that topic.The default value is `false`.                                                                                                                                                                                                                  |
-| connectionSelector | true     | reuse the connection to mqtt broker. [more info](../sources/mqtt.md#connectionselector)                                                                                                                                                                                                                                                                   | 
+| connectionSelector | true     | reuse the connection to mqtt broker. [more info](../../sources/builtin/mqtt.md#connectionselector)                                                                                                                                                                                                                                                        | 
 
 Below is sample configuration for connecting to Azure IoT Hub by using SAS authentication.
 ```json

docs/en_US/rules/sinks/overview.md

@@ -0,0 +1,29 @@
+# Available Sinks
+
+In the eKuiper source code, there are built-in sinks and sinks in extension.
+
+## Built-in Sinks
+
+Users can directly use the built-in sinks in the standard eKuiper instance. The list of built-in sinks are:
+
+- [Mqtt sink](./builtin/mqtt.md): sink to external mqtt broker
+- [EdgeX sink](./builtin/edgex.md): sink to EdgeX Foundry. This sink only exist when enabling edgex build tag.
+- [Rest sink](./builtin/rest.md): sink to external http server.
+- [Memory sink](./builtin/memory.md): sink to eKuiper memory topic to form rule pipelines.
+- [Log sink](./builtin/log.md): sink to log, usually for debug only.
+- [Nop sink](./builtin/nop.md): sink to nowhere. It is used for performance testing now.
+
+## Predefined Sink Plugins
+
+We have developed some official sink plugins. These plugins can be found in eKuiper's source code and users need to build them manually. Please check each sink about how to build and use.
+
+Additionally, these plugins have pre-built binaries for the mainstream cpu architecture such as AMD or ARM. The pre-built plugin hosted in `https://packages.emqx.net/kuiper-plugins/$version/$os/sinks/$type_$arch.zip`. For example, to get tdengine sink for debian amd64, install it from `https://packages.emqx.net/kuiper-plugins/1.4.4/debian/sinks/tdengine_amd64.zip`.
+
+The list of predefined sink plugins:
+
+- [Zero MQ sink](./plugin/zmq.md): sink to zero mq.
+- [File sink](./plugin/file.md): sink to a file.
+- [InfluxDB sink](./plugin/influx.md): sink to influx db.
+- [Tdengine sink](./plugin/tdengine.md): sink to tdengine.
+- [Redis sink](./plugin/redis.md): sink to redis.
+- [Image sink](./plugin/image.md): sink to an image file. Only used to handle binary result.

docs/en_US/rules/sources/edgex.md

@@ -17,7 +17,7 @@ EdgeX source will try to get the data type of fields,
 - or keep original value if  field of a type can not be found in the readings's ValueType field;
 - or if failed to conver the value, then the value will be **dropped**, and a warning message print in the log;
 
-The types defined in readings will be converted into related [data types](../../sqls/streams.md) that supported in eKuiper.
+The types defined in readings will be converted into related [data types](../../../sqls/streams.md) that supported in eKuiper.
 
 #### Boolean
 
@@ -189,7 +189,7 @@ demo1: #Conf_key
   topic: events
 ```
 
-If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `demo1`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../sqls/streams.md) for more info).
+If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `demo1`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../../sqls/streams.md) for more info).
 
 **Sample**
 

docs/en_US/rules/sources/file.md

@@ -1,6 +1,6 @@
 ## File source
 
-eKuiper provides built-in support for reading file content into the eKuiper processing pipeline. The file source is usually used as a [table](../../sqls/tables.md) and it is the default type for create table statement.
+eKuiper provides built-in support for reading file content into the eKuiper processing pipeline. The file source is usually used as a [table](../../../sqls/tables.md) and it is the default type for create table statement.
 
 ```sql
 create table table1 (
@@ -10,7 +10,7 @@ create table table1 (
 ) WITH (DATASOURCE="lookup.json", FORMAT="json", TYPE="file");
 ```
 
-You can use [cli](../../operation/cli/tables.md) or [rest api](../../operation/restapi/tables.md) to manage the tables.
+You can use [cli](../../../operation/cli/tables.md) or [rest api](../../../operation/restapi/tables.md) to manage the tables.
 
 The configure file for the file source is in */etc/sources/file.yaml* in which the path to the file can be specified.
 

docs/en_US/rules/sources/http_pull.md

@@ -86,7 +86,7 @@ The HTTP request headers that you want to send along with the HTTP request.
 
 ## Override the default settings
 
-If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `application_conf`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../sqls/streams.md) for more info).
+If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `application_conf`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../../sqls/streams.md) for more info).
 
 **Sample**
 

docs/en_US/rules/sources/memory.md

@@ -1,6 +1,6 @@
 # Memory Source
 
-Memory source is provided to consume events produced by the [memory sink](../sinks/memory.md) through topics. The topic is like pubsub topic such as mqtt, so that there could be multiple memory sinks which publish to the same topic and multiple memory sources which subscribe to the same topic. The typical usage for memory action is to form [rule pipelines](../rule_pipeline.md).
+Memory source is provided to consume events produced by the [memory sink](../../sinks/builtin/memory.md) through topics. The topic is like pubsub topic such as mqtt, so that there could be multiple memory sinks which publish to the same topic and multiple memory sources which subscribe to the same topic. The typical usage for memory action is to form [rule pipelines](../../rule_pipeline.md).
 
 There is no configuration properties. The topic is specified by the stream data source property like below examples:
 

docs/en_US/rules/sources/mqtt.md

@@ -148,7 +148,7 @@ Expected field type.
 
 ## Override the default settings
 
-If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `demo_conf`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../sqls/streams.md) for more info).
+If you have a specific connection that need to overwrite the default settings, you can create a customized section. In the previous sample, we create a specific setting named with `demo_conf`.  Then you can specify the configuration with option `CONF_KEY` when creating the stream definition (see [stream specs](../../../sqls/streams.md) for more info).
 
 **Sample**
 

docs/en_US/rules/sources/overview.md

@@ -0,0 +1,24 @@
+# Available Sources
+
+In the eKuiper source code, there are built-in sources and sources in extension.
+
+## Built-in Sources
+
+Users can directly use the built-in sources in the standard eKuiper instance. The list of built-in sources are:
+
+- [Mqtt source](./builtin/mqtt.md): read data from mqtt topics.
+- [EdgeX source](./builtin/edgex.md): read data from EdgeX foundry.
+- [Http pull source](./builtin/http_pull.md): source to pull data from http servers.
+- [Memory source](./builtin/memory.md): source to read from eKuiper memory topic to form rule pipelines.
+- [File source](./builtin/file.md): source to read from file, usually used as tables.
+
+## Predefined Source Plugins
+
+We have developed some official source plugins. These plugins can be found in eKuiper's source code and users need to build them manually. Please check each source about how to build and use.
+
+Additionally, these plugins have pre-built binaries for the mainstream cpu architecture such as AMD or ARM. The pre-built plugin hosted in `https://packages.emqx.net/kuiper-plugins/$version/$os/sources/$type_$arch.zip`. For example, to get zmq source for debian amd64, install it from `https://packages.emqx.net/kuiper-plugins/1.4.4/debian/sources/zmq_amd64.zip`.
+
+The list of predefined source plugins:
+
+- [Zero MQ source](./plugin/zmq.md): read data from zero mq.
+- [Random source](./plugin/random.md): a source to generate random data for testing.

docs/en_US/extension/native/functions/functions.md

@@ -1,6 +1,6 @@
 # Custom function
 
-eKuiper can customize functions. For the development, compilation and use of functions, please [see here](../develop/function.md).
+eKuiper can customize functions. For the development, compilation and use of functions, please [see here](../extension/native/develop/function.md).
 
 ## echo plugin
 

docs/en_US/sqls/streams.md

@@ -6,16 +6,16 @@ In eKuiper, each column or an expression has a related data type. A data type de
 
 Below is the list of data types supported.
 
-| #    | Data type | Description                                                  |
-| ---- | --------- | ------------------------------------------------------------ |
-| 1    | bigint    |                                                              |
-| 2    | float     |                                                              |
-| 3    | string    |                                                              |
-| 4    | datetime  |                                                 |
-| 5    | boolean   |                                                              |
-| 6    | bytea   |  A sequence of bytes to store binary data. If the stream format is "JSON", the bytea field must be a base64 encoded string |
-| 7    | array     | The array type, can be any simple types or array and type. |
-| 8    | struct    | The complex type.                                            |
+| #   | Data type | Description                                                                                                               |
+|-----|-----------|---------------------------------------------------------------------------------------------------------------------------|
+| 1   | bigint    |                                                                                                                           |
+| 2   | float     |                                                                                                                           |
+| 3   | string    |                                                                                                                           |
+| 4   | datetime  |                                                                                                                           |
+| 5   | boolean   |                                                                                                                           |
+| 6   | bytea     | A sequence of bytes to store binary data. If the stream format is "JSON", the bytea field must be a base64 encoded string |
+| 7   | array     | The array type, can be any simple types or array and type.                                                                |
+| 8   | struct    | The complex type.                                                                                                         |
 
 ## Language definitions
 
@@ -28,17 +28,17 @@ CREATE STREAM
 
 **The supported property names.**
 
-| Property name | Optional | Description                                                  |
-| ------------- | -------- | ------------------------------------------------------------ |
-| DATASOURCE | false    | The value is determined by source type. The topic names list if it's a MQTT data source. Please refer to related document for other sources. |
-| FORMAT        | true | The data format, currently the value can be "JSON" and "BINARY". The default is "JSON". Check [Binary Stream](#Binary Stream) for more detail. |
-| KEY           | true     | Reserved key, currently the field is not used. It will be used for GROUP BY statements. |
-| TYPE     | true | The source type, if not specified, the value is "mqtt". |
-| StrictValidation     | true | To control validation behavior of message field against stream schema. See [Strict Validation](#Strict Validation) for more info. |
-| CONF_KEY | true | If additional configuration items are requied to be configured, then specify the config key here. See [MQTT stream](../rules/sources/mqtt.md) for more info. |
-| SHARED | true | Whether the source instance will be shared across all rules using this stream |
-| TIMESTAMP | true | The field to represent the event's timestamp. If specified, the rule will run with event time. Otherwise, it will run with processing time. Please refer to [timestamp management](./windows.md#timestamp-management) for details. |
-| TIMESTAMP_FORMAT | true | The default format to be used when converting string to or from datetime type. |
+| Property name    | Optional | Description                                                                                                                                                                                                                        |
+|------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| DATASOURCE       | false    | The value is determined by source type. The topic names list if it's a MQTT data source. Please refer to related document for other sources.                                                                                       |
+| FORMAT           | true     | The data format, currently the value can be "JSON" and "BINARY". The default is "JSON". Check [Binary Stream](#Binary Stream) for more detail.                                                                                     |
+| KEY              | true     | Reserved key, currently the field is not used. It will be used for GROUP BY statements.                                                                                                                                            |
+| TYPE             | true     | The source type, if not specified, the value is "mqtt".                                                                                                                                                                            |
+| StrictValidation | true     | To control validation behavior of message field against stream schema. See [Strict Validation](#Strict Validation) for more info.                                                                                                  |
+| CONF_KEY         | true     | If additional configuration items are requied to be configured, then specify the config key here. See [MQTT stream](../rules/sources/builtin/mqtt.md) for more info.                                                               |
+| SHARED           | true     | Whether the source instance will be shared across all rules using this stream                                                                                                                                                      |
+| TIMESTAMP        | true     | The field to represent the event's timestamp. If specified, the rule will run with event time. Otherwise, it will run with processing time. Please refer to [timestamp management](./windows.md#timestamp-management) for details. |
+| TIMESTAMP_FORMAT | true     | The default format to be used when converting string to or from datetime type.                                                                                                                                                     |
 
 **Example 1,**
 
@@ -50,7 +50,7 @@ WITH ( datasource = "topic/temperature", FORMAT = "json", KEY = "id");
 
 The stream will subscribe to MQTT topic ``topic/temperature``, the server connection uses ``server`` key of ``default`` section in configuration file ``$ekuiper/etc/mqtt_source.yaml``. 
 
-- See [MQTT source](../rules/sources/mqtt.md) for more info.
+- See [MQTT source](../rules/sources/builtin/mqtt.md) for more info.
 
 **Example 2,**
 
@@ -67,7 +67,7 @@ demo (
 
 The stream will subscribe to MQTT topic `test/`, the server connection uses settings of `demo` section in configuration file `$ekuiper/etc/mqtt_source.yaml`. 
 
-- See [MQTT source](../rules/sources/mqtt.md) for more info.
+- See [MQTT source](../rules/sources/builtin/mqtt.md) for more info.
 
 - See [rules and streams CLI docs](../operation/cli/overview.md) for more information of rules & streams management.
 

docs/en_US/extension/native/functions/tensorflow_lite_tutorial.md

@@ -58,7 +58,7 @@ var LabelImage = labelImage{
 }
 ```
 
-Check [this tutorial](../develop/plugins_tutorial.md) for detail steps of creating eKuiper plugins.  Please refer to [labelImage.go](https://github.com/lf-edge/ekuiper/blob/master/extensions/functions/labelImage/labelImage.go) for the full source code.
+Check [this tutorial](../../extension/native/develop/plugins_tutorial.md) for detail steps of creating eKuiper plugins.  Please refer to [labelImage.go](https://github.com/lf-edge/ekuiper/blob/master/extensions/functions/labelImage/labelImage.go) for the full source code.
 
 ## Build and install the plugin
 

docs/en_US/deploy/openyurt_tutorial.md

@@ -1,7 +1,7 @@
 # Deploy and Manage eKuiper with OpenYurt
 
 LF Edge eKuiper is lightweight IoT data analytics and streaming software which is usually running in the edge side.
-A [manager dashboard](../operation/manager-ui/overview.md) is provided to manage one or multiple eKuiper instances. Typically, the
+A [manager dashboard](../../operation/manager-ui/overview.md) is provided to manage one or multiple eKuiper instances. Typically, the
 dashboard is deployed in a cloud node to manage eKuiper instances across many edge nodes.
 
 In most circumstances, the edge node is physically un-accessible from the cloud node due to security or other
@@ -320,7 +320,7 @@ healthy like the below screenshot:
 ![system](./ping.png)
 
 Great! Now we can manage the eKuiper in the edge by the dashboard, as if it was deployed in the cloud. Follow
-the [manager ui tutorial](../operation/manager-ui/overview.md) to create and manage your stream, rule and plugins and any other
+the [manager ui tutorial](../../operation/manager-ui/overview.md) to create and manage your stream, rule and plugins and any other
 management works of eKuiper from the cloud.
 
 ## Extended Reading
@@ -330,4 +330,4 @@ If you want to explore more features of eKuiper or OpenYurt, please refer to bel
 - [eKuiper Github code repository](https://github.com/lf-edge/ekuiper/)
 - [eKuiper reference guide](https://github.com/lf-edge/ekuiper/blob/edgex/docs/en_US/reference.md)
 - [OpenYurt tutorials](https://github.com/openyurtio/openyurt/tree/master/docs/tutorial)
-- [eKuiper manager ui tutorial](../operation/manager-ui/overview.md)
+- [eKuiper manager ui tutorial](../../operation/manager-ui/overview.md)