chore(ci): add build plugins on actions (#435)

.ci/Dockerfile-plugins

@@ -0,0 +1,19 @@
+FROM golang:1.14 AS builder
+
+ARG VERSION
+ARG PLUGIN_TYPE
+ARG PLUGIN_NAME
+
+COPY . /go/kuiper
+
+WORKDIR /go/kuiper
+
+RUN apt update && apt install -y pkg-config libczmq-dev jq zip
+
+RUN set -e -u -x \
+    && mkdir -p _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
+    && for lib in $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs[]"); do go get $lib; done \
+    && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
+    && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
+    && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
+    && zip -r ${PLUGIN_NAME}_$(go version -v | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .

.ci/Dockerfile-plugins-alpine

@@ -0,0 +1,19 @@
+FROM golang:1.14-alpine AS builder
+
+ARG VERSION
+ARG PLUGIN_TYPE
+ARG PLUGIN_NAME
+
+COPY . /go/kuiper
+
+WORKDIR /go/kuiper
+
+RUN apk add gcc make git libc-dev binutils-gold pkgconfig zeromq-dev jq zip
+
+RUN set -e -u -x \
+    && mkdir -p _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
+    && for lib in $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs[]"); do go get $lib; done \
+    && go build --buildmode=plugin -o _plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so plugins/$PLUGIN_TYPE/$PLUGIN_NAME.go \
+    && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml _plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
+    && cd _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
+    && zip -r ${PLUGIN_NAME}_$(go version -v | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .

.github/workflows/build_packages.yaml

@@ -60,15 +60,21 @@ jobs:
             name: packages-mac
             path: _packages/.
 
-    build-docker-images:
+    build-docker-images-and-plugins:
         runs-on: ubuntu-latest
-        
+
         strategy:
             matrix:
-                suffix:
-                - fat
-                - slim
-                - alpine
+                plugin:
+                - sinks/file
+                - sinks/influxdb
+                - sinks/taos
+                - sinks/zmq
+                - sources/random
+                - sources/zmq
+                - functions/accumulateWordCount
+                - functions/countPlusOne
+                - functions/echo
 
         steps:
         - uses: actions/checkout@v1
@@ -91,24 +97,74 @@ jobs:
             docker buildx create --use --name mybuild
             docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
         - name: build docker image
-          if: matrix.suffix == 'fat'
-          run: docker build --no-cache -t emqx/kuiper:$(git describe --tags --always) -f deploy/docker/Dockerfile .
-
-        - name: build docker image
-          if: matrix.suffix != 'fat'
+          run: make docker -j4
+        - name: buiild debian plugins
           env:
-              SUFFIX: ${{ matrix.suffix }}
-          run: docker build --no-cache -t emqx/kuiper:$(git describe --tags --always)-$SUFFIX -f deploy/docker/Dockerfile-$SUFFIX .
-
-        - name: test docker image
+            PLUGIN: ${{ matrix.plugin }}
+          run: make ${PLUGIN}
+        - name: buiild alpine plugins
+          env:
+            PLUGIN: ${{ matrix.plugin }}
+          run: make alpine/${PLUGIN}
+        - name: test docker and plugins
+          env:
+            PLUGIN: ${{ matrix.plugin }}
           run: |
             set -e -x -u
-            image_id=$(docker images emqx/kuiper -q | head -1)
-            container_id=$(docker run -d $image_id)
-            ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
-            sleep 10
-            if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
+            plugin_type=$(echo ${PLUGIN%%/*})
+            plugin_name=$(echo ${PLUGIN##*/})
+
+            for image_id in $(docker images emqx/kuiper -q); do
+                container_id=$(docker run -d -v $(pwd)/_plugins:/var/plugins $image_id)
+                ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
+                os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
+
+                sleep 5
+                if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
+                curl ${ip_address}:9081/plugins/${plugin_type} -X POST -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
+                [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
+            done
+        - uses: actions/upload-artifact@v2
+          with:
+            name: plugins
+            path: "_plugins/"
+
+    build-docker-manifest-images:
+        runs-on: ubuntu-latest
+        
+
+        needs:
+        - build-docker-images-and-plugins
+
+        strategy:
+            matrix:
+                suffix:
+                - fat
+                - slim
+                - alpine
 
+        steps:
+        - uses: actions/checkout@v1
+        - name: install docker
+          if: github.event_name == 'release'
+          run: |
+            sudo apt-get remove docker docker-engine docker.io containerd runc
+            sudo apt-get update
+            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+            sudo apt-get update
+            sudo apt-get install docker-ce docker-ce-cli containerd.io jq
+        - name: prepare docker
+          if: github.event_name == 'release'
+          run: |
+            mkdir -p $HOME/.docker
+            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
+            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
+            sudo systemctl restart docker
+            docker version
+            docker buildx create --use --name mybuild
+            docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
         - name: cross build docker images
           if: (matrix.suffix == 'fat') && (github.event_name == 'release')
           run: |
@@ -119,7 +175,6 @@ jobs:
             -t emqx/kuiper:$(git describe --tags --always) \
             -f deploy/docker/Dockerfile . \
             --push
-
         - name: cross build docker images
           if: (matrix.suffix != 'fat') && (github.event_name == 'release')
           env:
@@ -177,7 +232,10 @@ jobs:
     release:
         runs-on: ubuntu-latest
 
-        needs: [build, build-on-mac]
+        needs:
+        - build
+        - build-on-mac
+        - build-docker-images-and-plugins
 
         steps:
         - uses: actions/checkout@v1
@@ -189,6 +247,10 @@ jobs:
           with:
             name: packages-mac
             path: _packages
+        - uses: actions/download-artifact@v1
+          with:
+            name: plugins
+            path: _plugins
         - name: check packages
           run: |
             cd _packages && for var in $( ls |grep -v sha256); do
@@ -213,7 +275,8 @@ jobs:
             aws configure set default.region us-west-2
             aws s3 rm --quiet --recursive s3://packages.emqx.io/kuiper/$version
             aws s3 cp --quiet --recursive ./_packages s3://packages.emqx.io/kuiper/$version
-            aws cloudfront create-invalidation --distribution-id E3TYD0WSP4S14P --paths "/kuiper/$version/*"
+            aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx.io/kuiper-plugins/$version
+            aws cloudfront create-invalidation --distribution-id E3TYD0WSP4S14P --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"
         - name: update emqx.io
           if: github.event_name == 'release'
           run: |

Makefile

@@ -169,7 +169,60 @@ cross_docker: cross_prepare
 	-f deploy/docker/Dockerfile-alpine . \
 	--push
 
+
+PLUGINS := sinks/file \
+	sinks/influxdb \
+	sinks/taos \
+	sinks/zmq \
+	sources/random \
+	sources/zmq \
+	functions/accumulateWordCount \
+	functions/countPlusOne \
+	functions/echo
+
+.PHONE: plugins
+plugins: cross_prepare $(PLUGINS)
+$(PLUGINS): PLUGIN_TYPE = $(word 1, $(subst /, , $@))
+$(PLUGINS): PLUGIN_NAME = $(word 2, $(subst /, , $@))
+$(PLUGINS):
+	@docker buildx build --no-cache \
+    --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
+    -t cross_build \
+    --build-arg VERSION=$(VERSION) \
+    --build-arg PLUGIN_TYPE=$(PLUGIN_TYPE)\
+    --build-arg PLUGIN_NAME=$(PLUGIN_NAME)\
+    --output type=tar,dest=/tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar \
+    -f .ci/Dockerfile-plugins .
+
+	@mkdir -p _plugins/debian/$(PLUGIN_TYPE)
+	@for arch in amd64 arm64 arm_v7 386 ppc64le; do \
+		tar -xvf /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar --wildcards "linux_$${arch}/go/kuiper/_plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip" \
+		&& mv $$(ls linux_$${arch}/go/kuiper/_plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip) _plugins/debian/$(PLUGIN_TYPE); \
+	done
+	@rm -f /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar
+
+.PHONE: apline_plugins
+alpine_plugins: cross_prepare $(PLUGINS:%=alpine/%)
+$(PLUGINS:%=alpine/%): PLUGIN_TYPE = $(word 2, $(subst /, , $@))
+$(PLUGINS:%=alpine/%): PLUGIN_NAME = $(word 3, $(subst /, , $@))
+$(PLUGINS:%=alpine/%):
+	@docker buildx build --no-cache \
+    --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
+    -t cross_build \
+    --build-arg VERSION=$(VERSION) \
+    --build-arg PLUGIN_TYPE=$(PLUGIN_TYPE)\
+    --build-arg PLUGIN_NAME=$(PLUGIN_NAME)\
+    --output type=tar,dest=/tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME)_on_alpine.tar \
+    -f .ci/Dockerfile-plugins .
+
+	@mkdir -p _plugins/alpine/$(PLUGIN_TYPE)
+	@for arch in amd64 arm64 arm_v7 386 ppc64le; do \
+		tar -xvf /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME)_on_alpine.tar --wildcards "linux_$${arch}/go/kuiper/_plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip" \
+		&& mv $$(ls linux_$${arch}/go/kuiper/_plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip) _plugins/alpine/$(PLUGIN_TYPE); \
+	done
+	@rm -f /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME)_on_alpine.tar
+
 .PHONY: clean
 clean:
 	@rm -rf cross_build.tar linux_amd64 linux_arm64 linux_arm_v7 linux_ppc64le linux_386
-	@rm -rf _build _packages 
+	@rm -rf _build _packages _plugins