chore(CI): better build plugins

Signed-off-by: zhanghongtong <rory-z@outlook.com>

.ci/Dockerfile

@@ -1,25 +0,0 @@
-# Copyright 2021 EMQ Technologies Co., Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM golang:1.15.1 AS builder
-
-COPY . /go/kuiper
-
-WORKDIR /go/kuiper
-
-RUN apt update \
-    && apt install -y zip upx pkg-config libczmq-dev build-essential debhelper
-
-RUN make pkg
-RUN .ci/test.sh

.ci/Dockerfile-plugins

@@ -1,57 +0,0 @@
-# Copyright 2021 EMQ Technologies Co., Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM golang:1.15.1 AS builder
-
-ARG VERSION
-ARG PLUGIN_TYPE
-ARG PLUGIN_NAME
-
-COPY . /go/kuiper
-
-WORKDIR /go/kuiper
-
-RUN set -e -u -x \
-    && apt-get update \
-    && apt-get install -y pkg-config libczmq-dev jq zip --assume-yes apt-utils
-
-RUN set -e -u -x \
-    && mkdir -p _plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
-    && for lib in $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs[]"); do go get $lib; done \
-    && case $PLUGIN_NAME in \
-         influxdb ) \
-           go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go \
-           ;; \
-         tdengine ) \
-           if [ "$(uname -m)" = "x86_64" ]; then \
-             wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-x64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz; \
-           fi; \
-           if [ "$(uname -m)" = "aarch64" ]; then \
-             wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-aarch64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz; \
-           fi; \
-           tar -zxvf /tmp/TDengine-client-2.2.0.5.tar.gz \
-           && cd TDengine-client-2.2.0.5 && ./install_client.sh && cd - \
-           && go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go \
-           ;; \
-         labelImage ) \
-           git clone -b v2.2.0-rc3 --depth 1 https://github.com/tensorflow/tensorflow.git /tensorflow; \
-           CGO_CFLAGS=-I/tensorflow CGO_LDFLAGS=-L/go/kuiper/extensions/functions/labelImage/lib go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/functions/labelImage/labelImage.so extensions/functions/labelImage/*.go \
-           ;; \
-         * ) \
-           go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/*.go \
-           ;; \
-       esac \
-    && if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml plugins/$PLUGIN_TYPE/$PLUGIN_NAME; fi \
-    && cd plugins/$PLUGIN_TYPE/$PLUGIN_NAME \
-    && zip -r ${PLUGIN_NAME}_$(go version | grep -o "linux/.*" | sed -r 's linux/(.*) \1 g').zip .

.github/workflows/build_packages.yaml

@@ -125,7 +125,11 @@ jobs:
         docker run -d --name ekuiper docker.io/lfedge/ekuiper
         ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ekuiper)
         sleep 5
-        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
+        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then
+            echo "docker image failed"
+            docker logs ekuiper
+            exit 1
+        fi
     - uses: docker/metadata-action@v3
       id: meta
       with:
@@ -175,6 +179,14 @@ jobs:
           - functions/image
           - functions/geohash
           - functions/labelImage
+        arch:
+          - linux/amd64
+          - linux/arm64
+        golang:
+          - 1.15.11
+        exclude:
+          - arch: linux/arm64
+            plugin: functions/labelImage
 
     steps:
     - uses: actions/checkout@v2
@@ -185,8 +197,17 @@ jobs:
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
-    - name: Build single platform image
+    - name: build debian plugins
+      run: |
+        docker run -i --rm \
+            -v $(pwd):/ekuiper \
+            --workdir /ekuiper \
+            --platform ${{ matrix.arch }} \
+            ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-debian \
+            bash -euc "make ${{ matrix.plugin }}"
+    - name: Build ekuiper image
       uses: docker/build-push-action@v2
+      if: matrix.arch == 'linux/amd64'
       with:
         context: .
         platforms: linux/amd64
@@ -194,12 +215,8 @@ jobs:
         load: true
         tags: docker.io/lfedge/ekuiper
         file: deploy/docker/Dockerfile
-    - name: build debian plugins
-      env:
-        PLUGIN: ${{ matrix.plugin }}
-      run: |
-          make ${PLUGIN}
     - name: test docker and plugins
+      if: matrix.arch == 'linux/amd64'
       env:
         PLUGIN: ${{ matrix.plugin }}
       run: |

.gitignore

@@ -27,6 +27,7 @@ node_modules/
 
 _build
 _packages
+_plugins
 jmeter_logs
 kuiper_conf_util
 corss_build.tar
@@ -35,4 +36,4 @@ corss_build_for_rpm.tar
 *.swp
 *.history
 
-*/**/*.db
+*/**/*.db

Makefile

@@ -2,7 +2,9 @@ BUILD_PATH ?= _build
 PACKAGES_PATH ?= _packages
 
 VERSION := $(shell git describe --tags --always)
-PACKAGE_NAME := kuiper-$(VERSION)-$(shell go env GOOS)-$(shell go env GOARCH)
+ARCH := $(shell go env GOARCH)
+OS := $(shell go env GOOS)
+PACKAGE_NAME := kuiper-$(VERSION)-$(OS)-$(ARCH)
 
 TARGET ?= lfedge/ekuiper
 
@@ -65,10 +67,6 @@ real_pkg:
 	@mv $(BUILD_PATH)/$(PACKAGE_NAME).zip $(BUILD_PATH)/$(PACKAGE_NAME).tar.gz $(PACKAGES_PATH)
 	@echo "Package build success"
 
-.PHONY:cross_prepare
-cross_prepare:
-	@docker run --rm --privileged tonistiigi/binfmt --install all
-
 .PHONY: docker
 docker:
 	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION) -f deploy/docker/Dockerfile . --load
@@ -82,64 +80,21 @@ PLUGINS := sinks/file \
 	sinks/redis \
 	sources/random \
 	sources/zmq \
+	sinks/tdengine \
 	functions/accumulateWordCount \
 	functions/countPlusOne \
 	functions/image \
 	functions/geohash \
-	functions/echo
-
-.PHONY: plugins sinks/tdengine $(PLUGINS)
-plugins: cross_prepare sinks/tdengine functions/labelImage $(PLUGINS)
-sinks/tdengine:
-	@docker buildx build --no-cache \
-    --platform=linux/amd64,linux/arm64 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=sinks \
-    --build-arg PLUGIN_NAME=tdengine \
-    --output type=tar,dest=/tmp/cross_build_plugins_sinks_tdengine.tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/sinks
-	@for arch in amd64 arm64; do \
-		tar -xvf /tmp/cross_build_plugins_sinks_tdengine.tar --wildcards "linux_$${arch}/go/kuiper/plugins/sinks/tdengine/tdengine_$$(echo $${arch%%_*}).zip" \
-		&& mv $$(ls linux_$${arch}/go/kuiper/plugins/sinks/tdengine/tdengine_$$(echo $${arch%%_*}).zip) _plugins/debian/sinks; \
-	done
-	@rm -f /tmp/cross_build_plugins_sinks_tdengine.tar
-
-functions/labelImage:
-	@docker buildx build --no-cache \
-    --platform=linux/amd64 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=functions \
-    --build-arg PLUGIN_NAME=labelImage \
-    --output type=tar,dest=/tmp/cross_build_plugins_functions_labelImage.tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/functions
-	@tar -xvf /tmp/cross_build_plugins_functions_labelImage.tar --wildcards "go/kuiper/plugins/functions/labelImage/labelImage_amd64.zip"
-	@mv $$(ls go/kuiper/plugins/functions/labelImage/labelImage_amd64.zip) _plugins/debian/functions
-	@rm -f /tmp/cross_build_plugins_functions_labelImage.tar
+	functions/echo \
+	functions/labelImage
+
+.PHONY: plugins $(PLUGINS)
+plugins: $(PLUGINS)
 
 $(PLUGINS): PLUGIN_TYPE = $(word 1, $(subst /, , $@))
 $(PLUGINS): PLUGIN_NAME = $(word 2, $(subst /, , $@))
 $(PLUGINS):
-	@docker buildx build --no-cache \
-    --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
-    -t cross_build \
-    --build-arg VERSION=$(VERSION) \
-    --build-arg PLUGIN_TYPE=$(PLUGIN_TYPE)\
-    --build-arg PLUGIN_NAME=$(PLUGIN_NAME)\
-    --output type=tar,dest=/tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar \
-    -f .ci/Dockerfile-plugins .
-
-	@mkdir -p _plugins/debian/$(PLUGIN_TYPE)
-	@for arch in amd64 arm64 arm_v7 386; do \
-		tar -xvf /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar --wildcards "linux_$${arch}/go/kuiper/plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip" \
-		&& mv $$(ls linux_$${arch}/go/kuiper/plugins/$(PLUGIN_TYPE)/$(PLUGIN_NAME)/$(PLUGIN_NAME)_$$(echo $${arch%%_*}).zip) _plugins/debian/$(PLUGIN_TYPE); \
-	done
-	@rm -f /tmp/cross_build_plugins_$(PLUGIN_TYPE)_$(PLUGIN_NAME).tar
+	@$(CURDIR)/build-plugins.sh $(PLUGIN_TYPE) $(PLUGIN_NAME)
 
 .PHONY: clean
 clean:

build-plugins.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -euo pipefail
+
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
+
+PLUGIN_TYPE=$1
+PLUGIN_NAME=$2
+VERSION=$(git describe --tags --always)
+
+pre(){
+    mkdir -p _plugins/debian/$PLUGIN_TYPE
+    if [  $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs") != 'null' ]; then
+        for lib in $(cat etc/$PLUGIN_TYPE/$PLUGIN_NAME.json | jq -r ".libs[]"); do
+            go get $lib;
+        done
+    fi
+}
+
+post(){
+    if [ -f "etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml" ]; then
+        cp etc/$PLUGIN_TYPE/$PLUGIN_NAME.yaml extensions/$PLUGIN_TYPE/$PLUGIN_NAME;
+    fi
+    cd extensions/$PLUGIN_TYPE/$PLUGIN_NAME
+    zip -r ${PLUGIN_NAME}_$(go env GOARCH).zip .
+    cd -
+    mv $(find extensions/$PLUGIN_TYPE/$PLUGIN_NAME -name "*.zip")  _plugins/debian/$PLUGIN_TYPE
+}
+
+build(){
+    case $PLUGIN_NAME in
+        influx )
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go
+            ;;
+        tdengine )
+            if [ "$(uname -m)" = "x86_64" ]; then
+                wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-x64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz;
+            fi;
+            if [ "$(uname -m)" = "aarch64" ]; then
+                wget "https://www.taosdata.com/assets-download/TDengine-client-2.2.0.5-Linux-aarch64.tar.gz" -O /tmp/TDengine-client-2.2.0.5.tar.gz;
+            fi;
+            tar -zxvf /tmp/TDengine-client-2.2.0.5.tar.gz
+            cd TDengine-client-2.2.0.5 && ./install_client.sh && cd -
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -tags plugins -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME.go
+            ;;
+        labelImage )
+            git clone -b v2.2.0-rc3 --depth 1 https://github.com/tensorflow/tensorflow.git /tmp/tensorflow;
+            CGO_CFLAGS=-I/tmp/tensorflow CGO_LDFLAGS=-L$(pwd)/extensions/functions/labelImage/lib go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/functions/labelImage/labelImage.so extensions/functions/labelImage/*.go
+            ;;
+        * )
+            go build -trimpath -modfile extensions.mod --buildmode=plugin -o extensions/$PLUGIN_TYPE/$PLUGIN_NAME/$PLUGIN_NAME@$VERSION.so extensions/$PLUGIN_TYPE/$PLUGIN_NAME/*.go
+          ;;
+    esac
+}
+
+pre
+build
+post