ekuiper-new/.github/workflows/build_packages.yaml

name: Build packages

on:
  push:
    tags:
  pull_request:
  release:
    types:
      - published

jobs:
    build:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50 }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker info
            docker buildx create --use --name mybuild
        - name: build
          run: |
            make all_pkgs
            cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages
            path: _packages/.

    build-on-mac:
        runs-on: macos-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: actions/setup-go@v2
          with:
            go-version: '1.15.1'
        - name: prepare
          run: |
              brew install curl zip unzip gnu-sed upx pkg-config zmq
              echo "/usr/local/bin:$PATH" >> ~/.bashrc
        - name: build
          run: |
            make pkg
            cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages-mac
            path: _packages/.

    build-docker-images:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged tonistiigi/binfmt --install all
        - name: build docker image
          run: |
            make docker -j4
            docker save emqx/kuiper:$(git describe --tags --always)-slim > kuiper-image.tar.gz
        - name: test docker image
          run: |
            for image_id in $(docker images emqx/kuiper -q); do
              container_id=$(docker run -d $image_id)
              ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
              sleep 5
              if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
            done
        - uses: actions/upload-artifact@v2
          with:
            name: kuiper-image
            path: "kuiper-image.tar.gz"

    build-plugins:
        runs-on: ubuntu-latest

        needs: build-docker-images

        strategy:
            fail-fast: false
            matrix:
                plugin:
                - sinks/file
                - sinks/image
                - sinks/influx
                - sinks/tdengine
                - sinks/zmq
                - sources/random
                - sources/zmq
                - functions/accumulateWordCount
                - functions/countPlusOne
                - functions/echo
                - functions/image
                - functions/geohash
                - functions/labelImage

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged tonistiigi/binfmt --install all
        - name: build debian plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: |
              make ${PLUGIN}
        - uses: actions/download-artifact@v1
          with:
            name: kuiper-image
            path: .
        - name: test docker and plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: |
            set -e -x -u
            docker load < kuiper-image.tar.gz
            plugin_type=$(echo ${PLUGIN%%/*})
            plugin_name=$(echo ${PLUGIN##*/})

            for image_id in $(docker images emqx/kuiper -q); do
                container_id=$(docker run -d -v $(pwd)/_plugins:/var/plugins $image_id)
                ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
                os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
                sleep 5
                if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
                if [ ${os} = alpine ]; then continue; fi
                if [ "${plugin_name}" = "tdengine" ]; then
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"shellParas\": [\"2.0.3.1\"]}"
                elif [ "${plugin_name}" = "image" ]; then
                     curl \
                     ${ip_address}:9081/plugins/${plugin_type} \
                     -X POST \
                     -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"resize\",\"thumbnail\"]}"
                elif [ "${plugin_name}" = "geohash" ]; then
                     curl \
                     ${ip_address}:9081/plugins/${plugin_type} \
                     -X POST \
                     -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"geohashEncode\", \"geohashEncodeInt\", \"geohashDecode\", \"geohashDecodeInt\", \"geohashBoundingBox\", \"geohashBoundingBoxInt\", \"geohashNeighbor\", \"geohashNeighborInt\", \"geohashNeighbors\", \"geohashNeighborsInt\"]}"
                else
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
                fi
                docker logs ${container_id}
                [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
            done
        - uses: actions/upload-artifact@v2
          with:
            name: plugins
            path: "_plugins/"

    build-docker-manifest-images:
        runs-on: ubuntu-latest

        needs: build-docker-images

        if: github.event_name == 'release'

        strategy:
            matrix:
                suffix:
                - fat
                - slim
                - alpine

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged tonistiigi/binfmt --install all
        - name: cross build docker images
          if: matrix.suffix == 'fat'
          run: |
            set -e -u -x
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
            -t emqx/kuiper:$(git describe --tags --always) \
            -f deploy/docker/Dockerfile . \
            --push
        - name: cross build docker images
          if: matrix.suffix != 'fat'
          env:
              SUFFIX: ${{ matrix.suffix }}
          run: |
            set -e -u -x
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
            -t emqx/kuiper:$(git describe --tags --always)-$SUFFIX \
            -f deploy/docker/Dockerfile-$SUFFIX . \
            --push

    build-kubernetes-tool:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - name: install docker
          run: |
            sudo apt-get remove docker docker-engine docker.io containerd runc
            sudo apt-get update
            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
            sudo apt-get update
            sudo apt-get install docker-ce docker-ce-cli containerd.io
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged tonistiigi/binfmt --install all
        - name: build docker image
          run: docker build --no-cache -t emqx/kuiper-kubernetes-tool:$(git describe --tags --always) -f deploy/docker/Dockerfile-kubernetes-tool .
        - name: test docker image
          run: |
            set -e -u -x
            docker run -d --name kuiper-kubernetes-tool emqx/kuiper-kubernetes-tool:$(git describe --tags --always)
            sleep 5
            if [ "$(docker logs kuiper-kubernetes-tool)" != "Kuiper kubernetes tool is started successfully!"  ]; then exit 1; fi
        - name: buildx docker image
          if: github.event_name == 'release'
          run: |
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
                --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
                -t emqx/kuiper-kubernetes-tool:$(git describe --tags --always) \
                -f deploy/docker/Dockerfile-kubernetes-tool . \
                --push

    release:
        runs-on: ubuntu-latest

        needs:
        - build
        - build-on-mac
        - build-plugins

        steps:
        - uses: actions/checkout@v2
        - uses: actions/download-artifact@v1
          with:
            name: packages
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: packages-mac
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: plugins
            path: _plugins
        - name: check packages
          run: |
            cd _packages && for var in $( ls |grep -v sha256); do
              echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
            done
        - uses: zhanghongtong/upload-release-asset@v1
          if: github.event_name == 'release'
          with:
            repo: kuiper
            path: "_packages/kuiper-*"
            token: ${{ secrets.AccessToken }}
        - name: create invalidation for cloudfront
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
            aws configure set default.region us-west-2
            aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version
            aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"
        - name: update emqx.io
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            curl -w %{http_code} \
               --insecure \
               -H "Content-Type: application/json" \
               -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
               -X POST \
               -d "{\"repo\":\"emqx/kuiper\", \"tag\": \"${version}\" }" \
               ${{ secrets.EMQX_IO_RELEASE_API }}
        - name: update helm packages
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            curl \
            -H "Authorization: token ${{ secrets.AccessToken }}" \
            -H "Accept: application/vnd.github.v3+json" \
            -X POST \
            -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${version}\", \"emqx_ce\": \"true\"}}" \
            https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/kuiper_repos.yaml/dispatches