chore: add docker image for kubernetes tool (#371)

* chore: add docker image for kubeedge tool

* chore(ci): update actions

.github/workflows/build_packages.yaml

@@ -1,7 +1,6 @@
 name: Build packages
 
 on:
-    push:
     pull_request:
     release:
         types:
@@ -106,7 +105,43 @@ jobs:
           run: |
             echo ${{ secrets.DockerHubPassword }} | docker login -u ${{ secrets.DockerHubUser }} --password-stdin
             make cross_docker
-      
+
+    build-kubernetes-tool:
+        runs-on: ubuntu-latest
+
+        steps:
+        - uses: actions/checkout@v1
+        - name: install docker
+          run: |
+            sudo apt-get remove docker docker-engine docker.io containerd runc
+            sudo apt-get update
+            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+            sudo apt-get update
+            sudo apt-get install docker-ce docker-ce-cli containerd.io
+        - name: prepare docker
+          run: |
+            mkdir -p $HOME/.docker
+            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
+            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
+            sudo systemctl restart docker
+            docker version
+            docker buildx create --use --name mybuild
+        - name: build docker image
+          run: docker build --no-cache -t emqx/kuiper-kubernetes-tool:$(git describe --tags --always) -f deploy/docker/Dockerfile-kubernetes-tool .
+        - name: test docker image
+          run: |
+            set -e -u -x
+            docker run -d --name kuiper-kubernetes-tool emqx/kuiper-kubernetes-tool:$(git describe --tags --always)
+            sleep 5
+            if [ "$(docker logs kuiper-kubernetes-tool)" != "Kuiper kubernetes tool is started successfully!"  ]; then exit 1; fi
+        - name: push docker image
+          if: github.event_name == 'release'
+          run: |
+            echo ${{ secrets.DockerHubPassword }} | docker login -u ${{ secrets.DockerHubUser }} --password-stdin
+            docker push emqx/kuiper-kubernetes-tool:$(git describe --tags --always)
+
     release:
         runs-on: ubuntu-latest
 

.github/workflows/run_fvt_tests.yaml

@@ -70,69 +70,6 @@ jobs:
                 echo "FVT tests error"
                 exit 1
             fi
-            
-    fvt_tests_without_edgex:
-        runs-on: ubuntu-latest
-
-        steps:
-        - uses: actions/setup-go@v1
-          with:
-            go-version: '1.14'
-        - uses: actions/setup-java@v1
-          with:
-            java-version: '8' # The JDK version to make available on the path.
-            java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
-            architecture: x64 # (x64 or x86) - defaults to x64
-        - name: install jmeter
-          timeout-minutes: 10
-          env:
-            JMETER_VERSION: 5.2.1
-          run: |
-            wget -O /tmp/apache-jmeter.tgz http://us.mirrors.quenda.co/apache//jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
-            cd /tmp && tar -xvf apache-jmeter.tgz
-            echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
-            echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
-            wget -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-1.13-jar-with-dependencies.jar https://github.com/emqx/mqtt-jmeter/raw/master/Download/v1.13.0/mqtt-xmeter-1.13-jar-with-dependencies.jar
-            ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
-        - name: install emqx
-          env:
-            EMQX_VERSION: v4.0.2
-          run: |
-            wget -O emqx.deb https://www.emqx.io/downloads/broker/v4.0.2/emqx-ubuntu18.04-${EMQX_VERSION}_amd64.deb
-            sudo dpkg -i emqx.deb
-        - uses: actions/checkout@v2
-        - name: build kuiper
-          run: |
-            sudo apt update && sudo apt install pkg-config libczmq-dev -y
-            make
-            go build --buildmode=plugin -o plugins/sources/Zmq.so plugins/sources/zmq.go
-        - name: run edgex && emqx && kuiper
-          run: |
-            sudo ./fvt_scripts/setup_env.sh
-            ln -s _build/kuiper-$(git describe --tags --always)-$(uname -s | tr "[A-Z]" "[a-z]")-x86_64/log kuiper_logs
-        - name: run fvt tests
-          timeout-minutes: 5
-          run: ./fvt_scripts/run_jmeter.sh with_edgex=false
-        - uses: actions/upload-artifact@v1
-          if: always()
-          with:
-            name: kuiper_logs_without_edgex
-            path: ./kuiper_logs
-        - uses: actions/upload-artifact@v1
-          if: always()
-          with:
-            name: jmeter_logs_without_edgex
-            path: ./jmeter_logs
-        - name: check logs
-          run: |
-            sudo apt update && sudo apt install -y libxml2-utils
-            cd jmeter_logs
-            if [ ! -z "$(cat *.jtl| grep '<failure>' | awk -F '>' '{print $2}' | awk -F '<' '{print $1}' | grep true)" ] ||
-               [ "$(xmllint --format --xpath '/testResults/sample/@rc' $(ls *.jtl) | sed -r 's/ /\n/g;' | sort -u | grep -E 'rc=\"[45][0-9][0-9]\"|rc=\"\"')" != "" ]; then
-                echo -e "---------------------------------------------\n"
-                echo "FVT tests error"
-                exit 1
-            fi
 
     fvt_tests_for_container_in_helm:
       runs-on: ubuntu-latest
@@ -263,4 +200,4 @@ jobs:
               echo -e "---------------------------------------------\n"
               echo "FVT tests error"
               exit 1
-          fi
+          fi

.github/workflows/run_test_case.yaml

@@ -27,13 +27,18 @@ jobs:
           run: apt-get update && apt-get install libzmq3-dev -y
         - name: run test case
           run: |
+            set -e -u -x
             mkdir -p data
             mkdir -p log
+            sed -i -r "s/debug: .*/debug: true/1" etc/kuiper.yaml
             go build --buildmode=plugin -o plugins/sources/Random@v2.0.0.so plugins/sources/random.go
             go build --buildmode=plugin -o plugins/sinks/File@v1.0.0.so plugins/sinks/file.go
             go build --buildmode=plugin -o plugins/functions/Echo.so plugins/functions/echo.go
             go build --buildmode=plugin -o plugins/functions/CountPlusOne@v1.0.0.so plugins/functions/countPlusOne.go
             go build --buildmode=plugin -o plugins/functions/AccumulateWordCount@v1.0.0.so plugins/functions/accumulateWordCount.go
-            go test ./...
             go test --tags=edgex ./...
-    
+        - uses: actions/upload-artifact@v1
+          if: failure()
+          with:
+            name: stream.log
+            path: log/stream.log

.gitignore

@@ -29,3 +29,6 @@ jmeter_logs
 kuiper_conf_util
 corss_build.tar
 corss_build_for_rpm.tar
+
+*.swp
+*.history

deploy/docker/Dockerfile-alpine

@@ -9,7 +9,7 @@ RUN go build  -o kuiper_conf_util deploy/docker/conf_util.go \
     && if [ "$(uname -m)" == "x86_64" ]; then apk add upx && upx ./kuiper_conf_util; fi
 RUN make build_with_edgex
 
-FROM alpine:3.10
+FROM alpine:3.12
 
 COPY ./deploy/docker/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY --from=builder /go/kuiper/kuiper_conf_util /usr/bin/kuiper_conf_util
@@ -26,4 +26,4 @@ EXPOSE 9081 20498
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
-CMD ["./bin/server"]
+CMD ["./bin/server"]

deploy/docker/Dockerfile-kubernetes-tool

@@ -0,0 +1,19 @@
+FROM golang:1.14-alpine AS builder
+
+COPY . /go/kuiper
+
+WORKDIR /go/kuiper
+
+RUN go build -o tools/kubernetes/kuiper-kubernetes-tool tools/kubernetes/main.go
+
+FROM alpine:3.12
+
+WORKDIR /kuiper-kubernetes-tool
+
+RUN mkdir -p log
+
+COPY --from=builder /go/kuiper/tools/kubernetes/kuiper-kubernetes-tool .
+COPY --from=builder /go/kuiper/tools/kubernetes/etc etc
+COPY --from=builder /go/kuiper/tools/kubernetes/sample sample
+
+CMD ["./kuiper-kubernetes-tool", "etc/cf.yaml"]

tools/kubernetes/README.md

@@ -34,7 +34,7 @@ commandDir: "./sample/" //Command folder path
 ```
 ### 1.4 Compile the program:
 
-Execute the command of `go build -o tools/kubernetes/kubernetes tools/kubernetes/main.go` to generate the kubernetes program.
+Execute the command of `go build -o tools/kubernetes/kuiper-kubernetes-tool tools/kubernetes/main.go` to generate the kubernetes program.
 
 ## 2 Example of stream operation