Browse Source

edgex: upgrade edgex lib version and go version (#532)

* edgex: upgrade edgex lib version and go version

* fix: upgrade to go 1.15

* fix: upgrade to go 1.15

* docs: fix docs issue #468

* refactor: change to Go 1.15.1

* refactor: change to Go 1.15.1
jinfahua 4 years ago
parent
commit
6e7fb56382

+ 1 - 1
.ci/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14 AS builder
+FROM golang:1.15.1 AS builder
 
 
 COPY . /go/kuiper
 COPY . /go/kuiper
 
 

+ 1 - 1
.ci/Dockerfile-centos

@@ -2,7 +2,7 @@ FROM centos:7
 
 
 RUN yum install -y make wget git which rpm-build gcc
 RUN yum install -y make wget git which rpm-build gcc
 
 
-ARG GO_VERSION=1.14.4
+ARG GO_VERSION=1.15.1
 
 
 RUN case $(arch) in \
 RUN case $(arch) in \
         x86_64) \
         x86_64) \

+ 2 - 2
.ci/Dockerfile-plugins

@@ -1,4 +1,4 @@
-FROM golang:1.14 AS builder
+FROM golang:1.15.1 AS builder
 
 
 ARG VERSION
 ARG VERSION
 ARG PLUGIN_TYPE
 ARG PLUGIN_TYPE
@@ -24,4 +24,4 @@ RUN set -e -u -x \
     && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
     && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
     && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
     && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
     && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
     && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
-    && zip -r ${PLUGIN_NAME}_$(go version -v | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .
+    && zip -r ${PLUGIN_NAME}_$(go version | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .

+ 2 - 2
.ci/Dockerfile-plugins-alpine

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine AS builder
+FROM golang:1.15.1-alpine AS builder
 
 
 ARG VERSION
 ARG VERSION
 ARG PLUGIN_TYPE
 ARG PLUGIN_TYPE
@@ -16,4 +16,4 @@ RUN set -e -u -x \
     && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
     && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
     && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
     && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
     && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
     && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
-    && zip -r ${PLUGIN_NAME}_$(go version -v | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .
+    && zip -r ${PLUGIN_NAME}_$(go version | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .

+ 1 - 1
.github/workflows/run_test_case.yaml

@@ -12,7 +12,7 @@ jobs:
         runs-on: ubuntu-latest
         runs-on: ubuntu-latest
 
 
         container:
         container:
-            image: golang:1.14
+            image: golang:1.15.1
 
 
         steps:
         steps:
         - uses: actions/checkout@v2
         - uses: actions/checkout@v2

+ 1 - 1
deploy/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14 AS builder
+FROM golang:1.15.1 AS builder
 
 
 COPY . /go/kuiper
 COPY . /go/kuiper
 
 

+ 1 - 1
deploy/docker/Dockerfile-alpine

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine AS builder
+FROM golang:1.15.1-alpine AS builder
 
 
 COPY . /go/kuiper
 COPY . /go/kuiper
 
 

+ 1 - 1
deploy/docker/Dockerfile-kubernetes-tool

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine AS builder
+FROM golang:1.15.1-alpine AS builder
 
 
 COPY . /go/kuiper
 COPY . /go/kuiper
 
 

+ 1 - 1
deploy/docker/Dockerfile-slim

@@ -1,4 +1,4 @@
-FROM golang:1.14 AS builder
+FROM golang:1.15.1 AS builder
 
 
 COPY . /go/kuiper
 COPY . /go/kuiper
 
 

+ 2 - 1
docs/en_US/edgex/edgex_rule_engine_tutorial.md

@@ -279,10 +279,11 @@ Current rule does not filter any data that are sent to Kuiper, so how to filter
 
 
 #### Extended Reading
 #### Extended Reading
 
 
+- Starting from Kuiper 0.9.1 version, [a visualized web UI](../manager-ui/overview.md) is released with a separated Docker image. You can manage the streams, rules and plugins through web page. 
 - Read [EdgeX source](../rules/sources/edgex.md) for more detailed information of configurations and data type conversion.
 - Read [EdgeX source](../rules/sources/edgex.md) for more detailed information of configurations and data type conversion.
 - [How to use meta function to extract additional data from EdgeX message bus?](edgex_meta.md) There are some other information are sent along with device service, such as event created time, event id etc. If you want to use such metadata information in your SQL statements, please refer to this doc.
 - [How to use meta function to extract additional data from EdgeX message bus?](edgex_meta.md) There are some other information are sent along with device service, such as event created time, event id etc. If you want to use such metadata information in your SQL statements, please refer to this doc.
 - [Use Golang template to customize analaysis result in Kuiper](../rules/data_template.md) Before the analysis result is sent to different sinks, the data template can be used to make more processing. You can refer to this doc for more scenarios of using data templates.
 - [Use Golang template to customize analaysis result in Kuiper](../rules/data_template.md) Before the analysis result is sent to different sinks, the data template can be used to make more processing. You can refer to this doc for more scenarios of using data templates.
-- [EdgeX message bus sink doc](../rules/sinks/edgex.md). The document describes how to use EdgeX message bus sink. If you'd like to send the analysis result into message bus, you are probably interested in this article. 
+- [EdgeX message bus sink doc](../rules/sinks/edgex.md). The document describes how to use EdgeX message bus sink. If you'd like to have your analysis result be consumed by other EdgeX services, you can send analysis data with EdgeX data format through this sink, and other EdgeX services can subscribe new message bus exposed by Kuiper sink.
 - [Kuiper plugin development tutorial](../plugins/plugins_tutorial.md): Kuiper plugin is based on the plugin mechanism of Golang, users can build loosely-coupled plugin applications,  dynamic loading and binding when it is running. You can refer to this article if you're interested in Kuiper plugin development.
 - [Kuiper plugin development tutorial](../plugins/plugins_tutorial.md): Kuiper plugin is based on the plugin mechanism of Golang, users can build loosely-coupled plugin applications,  dynamic loading and binding when it is running. You can refer to this article if you're interested in Kuiper plugin development.
 
 
  If you want to explore more features of EMQ X Kuiper, please refer to below resources.
  If you want to explore more features of EMQ X Kuiper, please refer to below resources.

+ 7 - 3
docs/en_US/rules/sinks/edgex.md

@@ -1,6 +1,10 @@
 # EdgeX Message Bus action
 # EdgeX Message Bus action
 
 
-The action is used for publish output message into EdgeX message bus.
+The action is used for publishing output message into EdgeX message bus.  
+
+**Please notice that, if you're using the ZeorMQ message bus, the action will create a NEW EdgeX message bus (with the address where running Kuiper service), but not by leveraging the original message bus (normally it's the address & port exposed by application service). **
+
+**Also, you need to expose the port number to host server before running the Kuiper server if you want to have the service available to other hosts.**
 
 
 | Property name | Optional | Description                                                  |
 | Property name | Optional | Description                                                  |
 | ------------- | -------- | ------------------------------------------------------------ |
 | ------------- | -------- | ------------------------------------------------------------ |
@@ -32,7 +36,7 @@ Below optional configurations are supported, please check MQTT specification for
 
 
 ## Examples
 ## Examples
 
 
-### Publish result to EdgeX message bus without keeping original metadata
+### Publish result to a new EdgeX message bus without keeping original metadata
 In this case, the original metadata value (such as ``id, pushed, created, modified, origin`` in ``Events`` structure, and ``id, created, modified, origin, pushed, device`` in ``Reading`` structure will not be kept). Kuiper acts as another EdgeX micro service here, and it has own ``device name``. A ``deviceName`` property is provided, and allows user to specify the device name of Kuiper. Below is one example,
 In this case, the original metadata value (such as ``id, pushed, created, modified, origin`` in ``Events`` structure, and ``id, created, modified, origin, pushed, device`` in ``Reading`` structure will not be kept). Kuiper acts as another EdgeX micro service here, and it has own ``device name``. A ``deviceName`` property is provided, and allows user to specify the device name of Kuiper. Below is one example,
 
 
 1) Data received from EdgeX message bus ``events`` topic,
 1) Data received from EdgeX message bus ``events`` topic,
@@ -81,7 +85,7 @@ Please notice that,
 - The device name of ``Event`` structure is changed to ``kuiper``
 - The device name of ``Event`` structure is changed to ``kuiper``
 - All of metadata for ``Events and Readings`` structure will be updated with new value. ``Created`` field is updated to another value generated by Kuiper (here is ``0``).
 - All of metadata for ``Events and Readings`` structure will be updated with new value. ``Created`` field is updated to another value generated by Kuiper (here is ``0``).
 
 
-### Publish result to EdgeX message bus with keeping original metadata
+### Publish result to a new EdgeX message bus with keeping original metadata
 But for some scenarios, you may want to keep some of original metadata. Such as keep the device name as original value that published to Kuiper (``demo`` in the sample), and also other metadata of readings arrays. In such case, Kuiper is acting as a filter - to filter NOT concerned messages, but still keep original data.
 But for some scenarios, you may want to keep some of original metadata. Such as keep the device name as original value that published to Kuiper (``demo`` in the sample), and also other metadata of readings arrays. In such case, Kuiper is acting as a filter - to filter NOT concerned messages, but still keep original data.
 
 
 Below is an example,
 Below is an example,

+ 2 - 1
docs/zh_CN/edgex/edgex_rule_engine_tutorial.md

@@ -272,10 +272,11 @@ Connecting to 127.0.0.1:20498...
 
 
 #### 扩展阅读
 #### 扩展阅读
 
 
+- 从 Kuiper 0.9.1 版本开始,通过一个单独的 Docker 镜像提供了 [可视化 web 用户交互界面](../manager-ui/overview.md),您可以通过该 web 界面进行流、规则和插件等管理。
 - 阅读 [EdgeX 源](../rules/sources/edgex.md) 获取更多详细信息,以及类型转换等。
 - 阅读 [EdgeX 源](../rules/sources/edgex.md) 获取更多详细信息,以及类型转换等。
 - [如何使用 meta 函数抽取在 EdgeX 消息总线中发送的更多信息?](edgex_meta.md) 设备服务往总线上发送数据的时候,一些额外的信息也随之发送,比如时间创建时间,id 等。如果你想在 SQL 语句中使用这些信息,请参考这篇文章。
 - [如何使用 meta 函数抽取在 EdgeX 消息总线中发送的更多信息?](edgex_meta.md) 设备服务往总线上发送数据的时候,一些额外的信息也随之发送,比如时间创建时间,id 等。如果你想在 SQL 语句中使用这些信息,请参考这篇文章。
 - [Kuiper 中使用 Golang 模版 (template) 定制分析结果](../rules/data_template.md) 分析结果在发送给不同的 sink 之前,可以使用数据模版对结果进行二次处理,参考这片文章可以获取更多的关于数据模版的使用场景。
 - [Kuiper 中使用 Golang 模版 (template) 定制分析结果](../rules/data_template.md) 分析结果在发送给不同的 sink 之前,可以使用数据模版对结果进行二次处理,参考这片文章可以获取更多的关于数据模版的使用场景。
-- [EdgeX 消息总线目标](../rules/sinks/edgex.md). 该文档描述了如何使用 EdgeX 消息总线目标。如果你想把分析结果发送到消息总线中,你可能对此文章感兴趣。 
+- [EdgeX 消息总线目标](../rules/sinks/edgex.md). 该文档描述了如何使用 EdgeX 消息总线目标。如果想把你的分析结果被别的 EdgeX 服务消费,你可以通过这个 sink 发送 EdgeX 格式的数据,别的 EdgeX 服务可以通过这个 Kuiper sink 暴露出来的新的消息总线进行订阅。
 - [Kuiper 插件开发教程](../plugins/plugins_tutorial.md): Kuiper 插件机制基于 Go 语言的插件机制,使用户可以构建松散耦合的插件程序,在运行时动态加载和绑定,如果您对开发插件有兴趣,请参考该文章。
 - [Kuiper 插件开发教程](../plugins/plugins_tutorial.md): Kuiper 插件机制基于 Go 语言的插件机制,使用户可以构建松散耦合的插件程序,在运行时动态加载和绑定,如果您对开发插件有兴趣,请参考该文章。
 
 
 如想了解更多的 EMQ X Kuiper 的信息,请参考以下资源。
 如想了解更多的 EMQ X Kuiper 的信息,请参考以下资源。

+ 4 - 0
docs/zh_CN/rules/sinks/edgex.md

@@ -2,6 +2,10 @@
 
 
 该目标用于将消息发送到 EdgeX 消息总线上。
 该目标用于将消息发送到 EdgeX 消息总线上。
 
 
+**请注意,如果你使用的是 ZeorMQ 消息总线,那么该 sink 会创建一个新的 EdgeX 消息总线(绑定到 Kuiper 服务所运行的地址),而不是利用原来既有的消息总线(通常为 application 服务所暴露的地址和端口)。 **
+
+**另外,如果你需要在别的主机上对你的端口可以进行访问,你需要在开始运行 Kuiper 服务之前,把端口号映射到主机上。**
+
 | 名称        | 可选 | Description                                                  |
 | 名称        | 可选 | Description                                                  |
 | ----------- | -------- | ------------------------------------------------------------ |
 | ----------- | -------- | ------------------------------------------------------------ |
 | protocol    | true     | 协议,如未指定,使用缺省值 `tcp`.                             |
 | protocol    | true     | 协议,如未指定,使用缺省值 `tcp`.                             |

+ 3 - 3
go.mod

@@ -6,8 +6,8 @@ require (
 	github.com/benbjohnson/clock v1.0.0
 	github.com/benbjohnson/clock v1.0.0
 	github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e
 	github.com/buger/jsonparser v0.0.0-20191004114745-ee4c978eae7e
 	github.com/eclipse/paho.mqtt.golang v1.2.0
 	github.com/eclipse/paho.mqtt.golang v1.2.0
-	github.com/edgexfoundry/go-mod-core-contracts v0.1.59
-	github.com/edgexfoundry/go-mod-messaging v0.1.21
+	github.com/edgexfoundry/go-mod-core-contracts v0.1.80
+	github.com/edgexfoundry/go-mod-messaging v0.1.23
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/go-yaml/yaml v2.1.0+incompatible
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/uuid v1.1.1
 	github.com/google/uuid v1.1.1
@@ -21,4 +21,4 @@ require (
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
 )
 )
 
 
-go 1.13
+go 1.15

+ 1 - 1
xsql/funcs_misc.go

@@ -68,7 +68,7 @@ func convCall(name string, args []interface{}) (interface{}, bool) {
 				}
 				}
 			case "string":
 			case "string":
 				if v1, ok1 := args[0].(int); ok1 {
 				if v1, ok1 := args[0].(int); ok1 {
-					return string(v1), true
+					return fmt.Sprintf("%d", v1), true
 				} else if v1, ok1 := args[0].(float64); ok1 {
 				} else if v1, ok1 := args[0].(float64); ok1 {
 					return fmt.Sprintf("%g", v1), true
 					return fmt.Sprintf("%g", v1), true
 				} else if v1, ok1 := args[0].(string); ok1 {
 				} else if v1, ok1 := args[0].(string); ok1 {

+ 3 - 3
xstream/states/kv_store.go

@@ -55,7 +55,7 @@ func (s *KVStore) restore() error {
 		} else {
 		} else {
 			s.checkpoints = cs
 			s.checkpoints = cs
 			for _, c := range cs {
 			for _, c := range cs {
-				if b2, ok := s.db.Get(string(c)); ok {
+				if b2, ok := s.db.Get(fmt.Sprintf("%d", c)); ok {
 					if m, err := bytesToMap(b2.([]byte)); err != nil {
 					if m, err := bytesToMap(b2.([]byte)); err != nil {
 						return fmt.Errorf("invalid checkpoint data: %s", err)
 						return fmt.Errorf("invalid checkpoint data: %s", err)
 					} else {
 					} else {
@@ -100,7 +100,7 @@ func (s *KVStore) SaveCheckpoint(checkpointId int64) error {
 			if err != nil {
 			if err != nil {
 				return fmt.Errorf("save checkpoint err, fail to encode states: %s", err)
 				return fmt.Errorf("save checkpoint err, fail to encode states: %s", err)
 			}
 			}
-			err = s.db.Replace(string(checkpointId), b)
+			err = s.db.Replace(fmt.Sprintf("%d", checkpointId), b)
 			if err != nil {
 			if err != nil {
 				return fmt.Errorf("save checkpoint err: %v", err)
 				return fmt.Errorf("save checkpoint err: %v", err)
 			}
 			}
@@ -111,7 +111,7 @@ func (s *KVStore) SaveCheckpoint(checkpointId int64) error {
 				cp := s.checkpoints[0]
 				cp := s.checkpoints[0]
 				s.checkpoints = s.checkpoints[1:]
 				s.checkpoints = s.checkpoints[1:]
 				go func() {
 				go func() {
-					_ = s.db.Delete(string(cp))
+					_ = s.db.Delete(fmt.Sprintf("%d", cp))
 				}()
 				}()
 			}
 			}
 			cs, ok := sliceToBytes(s.checkpoints)
 			cs, ok := sliceToBytes(s.checkpoints)