ekuiper-new/.github/workflows/build_packages.yaml

name: Build packages

on:
  pull_request:
  release:
    types:
      - published

jobs:
    build:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v1
        - name: install docker
          run: |
            sudo apt-get remove docker docker-engine docker.io containerd runc
            sudo apt-get update
            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
            sudo apt-get update
            sudo apt-get install docker-ce docker-ce-cli containerd.io
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker info
            docker buildx create --use --name mybuild
        - name: build
          run: |
            make all_pkgs
            cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages
            path: _packages/.

    build-on-mac:
        runs-on: macos-latest

        steps:
        - uses: actions/checkout@v1
        - uses: actions/setup-go@v1
          with:
            go-version: '1.14'
        - name: prepare
          run: |
              /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
              brew install curl zip unzip gnu-sed upx pkg-config zmq
              echo "/usr/local/bin:$PATH" >> ~/.bashrc
        - name: build
          run: |
            make pkg
            cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages-mac
            path: _packages/.

    build-docker-images-and-plugins:
        runs-on: ubuntu-latest

        strategy:
            matrix:
                plugin:
                - sinks/file
                - sinks/influx
                - sinks/tdengine
                - sinks/zmq
                - sources/random
                - sources/zmq
                - functions/accumulateWordCount
                - functions/countPlusOne
                - functions/echo

        steps:
        - uses: actions/checkout@v1
        - name: install docker
          run: |
            sudo apt-get remove docker docker-engine docker.io containerd runc
            sudo apt-get update
            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
            sudo apt-get update
            sudo apt-get install docker-ce docker-ce-cli containerd.io
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
        - name: build docker image
          run: make docker -j4
        - name: buiild debian plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: make ${PLUGIN}
        - name: test docker and plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: |
            set -e -x -u
            plugin_type=$(echo ${PLUGIN%%/*})
            plugin_name=$(echo ${PLUGIN##*/})

            for image_id in $(docker images emqx/kuiper -q); do
                container_id=$(docker run -d -v $(pwd)/_plugins:/var/plugins $image_id)
                ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
                os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
                sleep 5
                if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
                if [ ${os} = alpine ]; then continue; fi
                if [ "${plugin_name}" = "tdengine" ]; then
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"shellParas\": [\"2.0.3.1\"]}"
                else
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
                fi
                docker logs ${container_id}
                [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
            done
        - uses: actions/upload-artifact@v2
          with:
            name: plugins
            path: "_plugins/"

    build-docker-manifest-images:
        runs-on: ubuntu-latest
        

        needs:
        - build-docker-images-and-plugins

        strategy:
            matrix:
                suffix:
                - fat
                - slim
                - alpine

        steps:
        - uses: actions/checkout@v1
        - name: install docker
          if: github.event_name == 'release'
          run: |
            sudo apt-get remove docker docker-engine docker.io containerd runc
            sudo apt-get update
            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
            sudo apt-get update
            sudo apt-get install docker-ce docker-ce-cli containerd.io jq
        - name: prepare docker
          if: github.event_name == 'release'
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
        - name: cross build docker images
          if: (matrix.suffix == 'fat') && (github.event_name == 'release')
          run: |
            set -e -u -x
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
            -t emqx/kuiper:$(git describe --tags --always) \
            -f deploy/docker/Dockerfile . \
            --push
        - name: cross build docker images
          if: (matrix.suffix != 'fat') && (github.event_name == 'release')
          env:
              SUFFIX: ${{ matrix.suffix }}
          run: |
            set -e -u -x
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/ppc64le \
            -t emqx/kuiper:$(git describe --tags --always)-$SUFFIX \
            -f deploy/docker/Dockerfile-$SUFFIX . \
            --push

    build-kubernetes-tool:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v1
        - name: install docker
          run: |
            sudo apt-get remove docker docker-engine docker.io containerd runc
            sudo apt-get update
            sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
            sudo apt-get update
            sudo apt-get install docker-ce docker-ce-cli containerd.io
        - name: prepare docker
          run: |
            mkdir -p $HOME/.docker
            echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
            echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50, "graph": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
            sudo systemctl restart docker
            docker version
            docker buildx create --use --name mybuild
            docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
        - name: build docker image
          run: docker build --no-cache -t emqx/kuiper-kubernetes-tool:$(git describe --tags --always) -f deploy/docker/Dockerfile-kubernetes-tool .
        - name: test docker image
          run: |
            set -e -u -x
            docker run -d --name kuiper-kubernetes-tool emqx/kuiper-kubernetes-tool:$(git describe --tags --always)
            sleep 5
            if [ "$(docker logs kuiper-kubernetes-tool)" != "Kuiper kubernetes tool is started successfully!"  ]; then exit 1; fi
        - name: buildx docker image
          if: github.event_name == 'release'
          run: |
            echo ${{ secrets.DOCKER_HUB_TOKEN }} | docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
            docker buildx build --no-cache \
                --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
                -t emqx/kuiper-kubernetes-tool:$(git describe --tags --always) \
                -f deploy/docker/Dockerfile-kubernetes-tool . \
                --push

    release:
        runs-on: ubuntu-latest

        needs:
        - build
        - build-on-mac
        - build-docker-images-and-plugins

        steps:
        - uses: actions/checkout@v1
        - uses: actions/download-artifact@v1
          with:
            name: packages
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: packages-mac
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: plugins
            path: _plugins
        - name: check packages
          run: |
            cd _packages && for var in $( ls |grep -v sha256); do
              echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
            done
        - name: update github release
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            for var in $(ls _packages) ; do
              .github/workflows/script/upload_github_release_asset.sh owner=emqx repo=kuiper tag=$version filename=_packages/$var github_api_token=$(echo ${{ secrets.AccessToken }})
            done
        - name: create invalidation for cloudfront
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            curl "https://d1vvhvl2y92vvt.cloudfront.net/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            sudo ./aws/install
            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
            aws configure set default.region us-west-2
            aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version
            aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"
        - name: update emqx.io
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            curl -w %{http_code} --insecure -H ${{ secrets.EmqxHeader }} https://admin.emqx.io/admin_api/v1/kuiper_github_release_callback?tag=$version
        - name: update helm packages
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            curl \
            -H "Authorization: token ${{ secrets.AccessToken }}" \
            -H "Accept: application/vnd.github.v3+json" \
            -X POST \
            -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${version}\", \"emqx_ce\": \"true\"}}" \
            https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/kuiper_repos.yaml/dispatches