chore(CI): update workflows

Signed-off-by: zhanghongtong <rory-z@outlook.com>

.ci/Dockerfile-centos

@@ -16,7 +16,7 @@ FROM centos:7
 
 RUN yum install -y make wget git which rpm-build gcc
 
-ARG GO_VERSION=1.15.1
+ARG GO_VERSION=1.15.11
 
 RUN case $(arch) in \
         x86_64) \
@@ -52,6 +52,3 @@ COPY . /go/kuiper
 WORKDIR /go/kuiper
 
 ENV KUIPER_SOURCE=/go/kuiper
-
-RUN make -C deploy/packages
-RUN .ci/test.sh

.github/dockerfile/Dockerfile.debian

@@ -0,0 +1,23 @@
+# Copyright 2021 EMQ Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG GO_VERSION=1.15.11
+FROM golang:${GO_VERSION} AS builder
+
+COPY . /go/kuiper
+
+WORKDIR /go/kuiper
+
+RUN apt update \
+    && apt install -y zip upx pkg-config libczmq-dev build-essential debhelper jq zip

.github/workflows/build_base_image.yaml

@@ -0,0 +1,46 @@
+name: Update docker base image
+
+## Update the base image every day
+on:
+  workflow_dispatch:
+  schedule:
+  ## In UTC
+  - cron: '0 0 * * *'
+
+## Build base images to be used by other github workflows
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - debian
+          - centos
+        golang:
+          - 1.15.11
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/setup-qemu-action@v1
+        with:
+          image: tonistiigi/binfmt:latest
+          platforms: all
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - uses: docker/build-push-action@v2
+        with:
+          push: true
+          pull: true
+          no-cache: true
+          platforms: linux/amd64,linux/arm64
+          build-args: GO_VERSION=${{ matrix.golang }}
+          tags: ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-${{ matrix.os }}
+          file: .github/dockerfile/Dockerfile.${{ matrix.os }}

.github/workflows/build_packages.yaml

@@ -12,6 +12,17 @@ jobs:
     build:
         runs-on: ubuntu-latest
 
+        strategy:
+            matrix:
+                golang:
+                - 1.15.11
+                arch:
+                - linux/amd64
+                - linux/arm64
+                os:
+                - debian
+                - centos
+
         steps:
         - uses: actions/checkout@v2
           with:
@@ -21,9 +32,30 @@ jobs:
           with:
             image: tonistiigi/binfmt:latest
             platforms: all
+        - name: get docker sha256
+          id: disgest
+          run: |
+              digest=$(docker buildx imagetools inspect ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-${{ matrix.os }} --raw | jq --raw-output '.manifests[] | select(.platform.architecture=="amd64").digest')
+              echo "::set-output name=image::ghcr.io/${{ github.repository }}/base@${digest}"
+        - run: docker pull ${{ steps.disgest.outputs.image }}
+        - name: build base image
+          uses: docker/build-push-action@v2
+          with:
+            pull: true
+            load: true
+            cache-from: ${{ steps.disgest.outputs.image }}
+            platforms: ${{ matrix.arch }}
+            build-args: GO_VERSION=${{ matrix.golang }}
+            tags: ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-${{ matrix.os }}
+            file: .github/dockerfile/Dockerfile.${{ matrix.os }}
         - name: build
           run: |
-            make all_pkgs
+            docker run -i --rm \
+            -v $(pwd):/ekuiper \
+            --workdir /ekuiper \
+            --platform ${{ matrix.arch }} \
+            ghcr.io/${{ github.repository }}/base:${{ matrix.golang }}-${{ matrix.os }} \
+            bash -euc "make pkg  && .github/scripts/test.sh"
             cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
         - uses: actions/upload-artifact@v1
           with:

Makefile

@@ -69,46 +69,6 @@ real_pkg:
 cross_prepare:
 	@docker run --rm --privileged tonistiigi/binfmt --install all
 
-.PHONY: cross_build
-cross_build: cross_prepare
-	@docker buildx build --no-cache \
-	--platform=linux/amd64,linux/arm64,linux/arm/v7 \
-	-t cross_build \
-	--output type=tar,dest=cross_build.tar \
-	-f .ci/Dockerfile .
-
-	@mkdir -p $(PACKAGES_PATH)
-	@tar -xvf cross_build.tar --wildcards linux_amd64/go/kuiper/_packages/ \
-		&& mv linux_amd64/go/kuiper/_packages/* $(PACKAGES_PATH)
-	@tar -xvf cross_build.tar --wildcards linux_arm64/go/kuiper/_packages/ \
-		&& mv linux_arm64/go/kuiper/_packages/* $(PACKAGES_PATH)
-	@tar -xvf cross_build.tar --wildcards linux_arm_v7/go/kuiper/_packages/ \
-		&& mv linux_arm_v7/go/kuiper/_packages/* $(PACKAGES_PATH)
-
-	@rm -f cross_build.tar
-	@echo "Cross build success"
-
-.PHONY: cross_build_for_rpm
-cross_build_for_rpm: cross_prepare
-	@docker buildx build --no-cache \
-	--platform=linux/amd64,linux/arm64 \
-	-t cross_build \
-	--output type=tar,dest=cross_build_for_rpm.tar \
-	-f .ci/Dockerfile-centos .
-
-	@mkdir -p $(PACKAGES_PATH)
-	@tar -xvf cross_build_for_rpm.tar --wildcards linux_amd64/go/kuiper/_packages/ \
-		&& mv linux_amd64/go/kuiper/_packages/*.rpm $(PACKAGES_PATH)
-	@tar -xvf cross_build_for_rpm.tar --wildcards linux_arm64/go/kuiper/_packages/ \
-		&& mv linux_arm64/go/kuiper/_packages/*.rpm $(PACKAGES_PATH)
-
-	@rm -f cross_build_for_rpm.tar
-	@echo "Cross build rpm packages success"
-
-
-.PHONE: all_pkgs
-all_pkgs: cross_build cross_build_for_rpm
-
 .PHONY: docker
 docker:
 	docker buildx build --no-cache --platform=linux/amd64 -t $(TARGET):$(VERSION) -f deploy/docker/Dockerfile . --load