ekuiper-new/.github/workflows/run_fvt_tests.yaml

name: Run fvt tests

concurrency:
  group: fvt-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    tags:
      - "*"
  pull_request:
  release:
    types:
      - published

jobs:
  fvt_tests_with_edgex:
    runs-on: ubuntu-latest

      # Service containers to run with `runner-job`
    services:
        # Label used to access the service container
      redis:
          # Docker Hub image
        image: redis
          # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
            # Maps port 6379 on service container to the host
          - 6379:6379
      emqx:
        image: emqx/emqx:4.3.10
        ports:
          - 1883:1883
          - 18083:18083

    steps:
    - uses: actions/setup-go@v1
      with:
        go-version: '1.17'
    - uses: actions/setup-java@v1
      with:
        java-version: '8' # The JDK version to make available on the path.
        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
        architecture: x64 # (x64 or x86) - defaults to x64
    - uses: actions/setup-python@v2  # For the test of python portable plugin
      with:
        python-version: '3.x'
        architecture: 'x64'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install pynng
    - name: install jmeter
      timeout-minutes: 10
      env:
          JMETER_VERSION: 5.4.3
      run: |
        wget --no-check-certificate -O /tmp/apache-jmeter.tgz https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
        cd /tmp && tar -xvf apache-jmeter.tgz
        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
        wget -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-1.13-jar-with-dependencies.jar https://github.com/emqx/mqtt-jmeter/raw/master/Download/v1.13.0/mqtt-xmeter-1.13-jar-with-dependencies.jar
        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
    - name: install tdengine client
      timeout-minutes: 10
      env:
        TD_VERSION: 2.4.0.18
      run: |
        if [ "$(uname -m)" = "x86_64" ]; then
          wget "https://www.taosdata.com/assets-download/TDengine-client-$TD_VERSION-Linux-x64.tar.gz" -O /tmp/TDengine-client-2.4.0.18.tar.gz;
        fi;
        if [ "$(uname -m)" = "aarch64" ]; then
          wget "https://www.taosdata.com/assets-download/TDengine-client-$TD_VERSION-Linux-aarch64.tar.gz" -O /tmp/TDengine-client-2.4.0.18.tar.gz;
        fi;
        tar -zxvf /tmp/TDengine-client-$TD_VERSION.tar.gz
        cd TDengine-client-$TD_VERSION && ./install_client.sh
    - uses: actions/checkout@v2
    - name: build kuiper
      run: |
        sudo apt update && sudo apt install pkg-config libczmq-dev -y
        make build_with_edgex
        go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/sources/Zmq.so extensions/sources/zmq/zmq.go
        go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/functions/Image.so extensions/functions/image/*.go
    - name: run edgex && kuiper
      run: |
        ./test/setup_env.sh
        ./test/prepare_plugins.sh
        ln -s _build/kuiper-$(git describe --tags --always)-$(uname -s | tr "[A-Z]" "[a-z]")-amd64/log kuiper_logs
    - name: run fvt tests
      timeout-minutes: 8
      run: ./test/run_jmeter.sh with_edgex=true
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: kuiper_logs_with_edgex
        path: ./kuiper_logs
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: jmeter_logs_with_edgex
        path: ./jmeter_logs
    - name: check logs
      run: |
        sudo apt update && sudo apt install -y libxml2-utils
        cd jmeter_logs
        for file in `ls *.jtl`
        do
          if [ ! -z "$(cat $file| grep '<failure>' | awk -F '>' '{print $2}' | awk -F '<' '{print $1}' | grep true)" ] ||
           [ "$(xmllint --format --xpath '/testResults/sample/@rc' $file | sed -r 's/ /\n/g;' | sort -u | grep -E 'rc=\"[45][0-9][0-9]\"|rc=\"\"')" != "" ]; then
            echo -e "---------------------------------------------\n"
            echo "FVT tests error for $file"
            exit 1
          fi
        done

  fvt_tests_redis_as_storage:
    runs-on: ubuntu-latest

    # Service containers to run with `runner-job`
    services:
        # Label used to access the service container
      redis:
          # Docker Hub image
        image: redis
          # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
            # Maps port 6379 on service container to the host
          - 6379:6379
      emqx:
        image: emqx/emqx:4.3.10
        ports:
          - 1883:1883
          - 18083:18083

    steps:
      - uses: actions/setup-go@v1
        with:
          go-version: '1.17'
      - uses: actions/setup-java@v1
        with:
          java-version: '8' # The JDK version to make available on the path.
          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
          architecture: x64 # (x64 or x86) - defaults to x64
      - name: install jmeter
        timeout-minutes: 10
        env:
          JMETER_VERSION: 5.4.3
        run: |
          wget --no-check-certificate -O /tmp/apache-jmeter.tgz https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
          cd /tmp && tar -xvf apache-jmeter.tgz
          echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
          echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
          wget -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-1.13-jar-with-dependencies.jar https://github.com/emqx/mqtt-jmeter/raw/master/Download/v1.13.0/mqtt-xmeter-1.13-jar-with-dependencies.jar
          ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
      - uses: actions/checkout@v2
      - name: build kuiper
        run: |
          sudo apt update && sudo apt install pkg-config libczmq-dev -y
          make build_with_edgex
          go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/sources/Zmq.so extensions/sources/zmq/zmq.go
          go build -trimpath -modfile extensions.mod --buildmode=plugin -o plugins/functions/Image.so extensions/functions/image/*.go
      - name: run edgex && kuiper
        env:
          KUIPER__STORE__TYPE: redis
          KUIPER__STORE__REDIS__PASSWORD: ""
        run: |
          ./test/setup_env.sh
          ln -s _build/kuiper-$(git describe --tags --always)-$(uname -s | tr "[A-Z]" "[a-z]")-amd64/log kuiper_logs
      - name: run fvt tests
        timeout-minutes: 8
        run: |
          rm -rf test/redis/set

          go build -o test/redis/set test/redis/set.go

          chmod +x test/redis/set
          fvt_dir=`pwd`
          /opt/jmeter/bin/jmeter.sh -Jjmeter.save.saveservice.output_format=xml -n -t test/redis_kv_storage.jmx -Dfvt="$fvt_dir" -l jmeter_logs/redis_kv_storage.jtl -j jmeter_logs/redis_kv_storage.log

      - uses: actions/upload-artifact@v1
        if: always()
        with:
          name: kuiper_logs_redis_storage
          path: ./kuiper_logs
      - uses: actions/upload-artifact@v1
        if: always()
        with:
          name: jmeter_logs_redis_storage
          path: ./jmeter_logs
      - name: check logs
        run: |
          sudo apt update && sudo apt install -y libxml2-utils
          cd jmeter_logs
          for file in `ls *.jtl`
          do
            if [ ! -z "$(cat $file| grep '<failure>' | awk -F '>' '{print $2}' | awk -F '<' '{print $1}' | grep true)" ] ||
             [ "$(xmllint --format --xpath '/testResults/sample/@rc' $file | sed -r 's/ /\n/g;' | sort -u | grep -E 'rc=\"[45][0-9][0-9]\"|rc=\"\"')" != "" ]; then
              echo -e "---------------------------------------------\n"
              echo "FVT tests error for $file"
              exit 1
            fi
          done

  fvt_tests_for_container_in_helm:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-java@v1
      with:
        java-version: '8' # The JDK version to make available on the path.
        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
        architecture: x64 # (x64 or x86) - defaults to x64
    - name: install jmeter
      timeout-minutes: 10
      env:
          JMETER_VERSION: 5.4.3
      run: |
        wget --no-check-certificate -O /tmp/apache-jmeter.tgz https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
        cd /tmp && tar -xvf apache-jmeter.tgz
        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
        wget -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-1.13-jar-with-dependencies.jar https://github.com/emqx/mqtt-jmeter/raw/master/Download/v1.13.0/mqtt-xmeter-1.13-jar-with-dependencies.jar
        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
    - name: setup jmeter
      timeout-minutes: 10
      run: |
        wget -O "/opt/jmeter/lib/json-lib-2.4-jdk15.jar" https://repo1.maven.org/maven2/net/sf/json-lib/json-lib/2.4/json-lib-2.4-jdk15.jar
        wget -O "/opt/jmeter/lib/commons-beanutils-1.8.0.jar" https://repo1.maven.org/maven2/commons-beanutils/commons-beanutils/1.8.0/commons-beanutils-1.8.0.jar
        wget -O "/opt/jmeter/lib/commons-collections-3.2.1.jar" https://repo1.maven.org/maven2/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar
        wget -O "/opt/jmeter/lib/commons-lang-2.5.jar" https://repo1.maven.org/maven2/commons-lang/commons-lang/2.5/commons-lang-2.5.jar
        wget -O "/opt/jmeter/lib/commons-logging-1.1.1.jar" https://repo1.maven.org/maven2/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar
        wget -O "/opt/jmeter/lib/ezmorph-1.0.6.jar" https://repo1.maven.org/maven2/net/sf/ezmorph/ezmorph/1.0.6/ezmorph-1.0.6.jar
    - name: install docker
      run: |
        sudo apt-get remove docker docker-engine docker.io containerd runc
        sudo apt-get update
        sudo apt-get install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
        sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
        sudo apt-get update
        sudo apt-get install docker-ce docker-ce-cli containerd.io
    - name: install k3s
      env:
        KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
      run: |
        sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts"
        curl -sfL https://get.k3s.io | sh -
        sudo chmod 644 /etc/rancher/k3s/k3s.yaml
        kubectl cluster-info
    - name: install helm
      run: |
        curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
        sudo chmod 700 get_helm.sh
        sudo ./get_helm.sh
        helm version
    - name: build kuiper for docker
      run: sudo docker build --no-cache -t lfedge/ekuiper:$(git describe --tags --alway)-alpine -f deploy/docker/Dockerfile-alpine .
    - name: run emqx on chart
      env:
        KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
      timeout-minutes: 5
      run: |
        helm repo add emqx https://repos.emqx.io/charts
        helm repo update
        helm install emqx --set replicaCount=1 emqx/emqx
        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "waiting emqx started"
          sleep 10
        done
    - name: run kuiper for chart
      env:
        KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
      timeout-minutes: 5
      run: |
        version=$(git describe --tags --always)
        emqx_address=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
        
        sudo docker save lfedge/ekuiper:$version-alpine -o kuier.tar.gz
        sudo k3s ctr image import kuier.tar.gz

        sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/chart/kuiper/Chart.yaml
        sed -i -r 's/  pullPolicy: .*$/  pullPolicy: Never/g' deploy/chart/kuiper/values.yaml
        sed -i -r "s/      server: .*$/      server: tcp:\/\/${emqx_address}:1883/g" deploy/chart/kuiper/values.yaml
        
        helm install kuiper deploy/chart/kuiper --debug --dry-run
        helm install kuiper deploy/chart/kuiper
        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=kuiper -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/name=kuiper -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "=============================="
          kubectl describe pods kuiper-0
          echo "=============================="
          kubectl get pods
          echo "=============================="
          echo "waiting kuiper started"
          sleep 10
        done
        kuiper_address=$(kubectl get svc --namespace default kuiper -o jsonpath="{.spec.clusterIP}")
        if [ $(curl -w %{http_code} -fsSL -o /dev/null $kuiper_address:9081) != 200 ];then
          kubectl logs kuiper-0
          exit 1;
        fi
    - name: check kuiper
      env:
        KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
      timeout-minutes: 5
      run: |
        emqx_address=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
        kuiper_address=$(kubectl get svc --namespace default kuiper -o jsonpath="{.spec.clusterIP}")
        /opt/jmeter/bin/jmeter.sh -Jjmeter.save.saveservice.output_format=xml -n -t test/select_aggr_rule.jmx -Dsrv=${kuiper_address} -Dmqtt_srv=${emqx_address} -l jmeter_logs/select_aggr_rule.jtl -j jmeter_logs/select_aggr_rule.log
        mkdir -p kuiper_logs
        kubectl exec kuiper-0 -- cat /kuiper/log/stream.log > kuiper_logs/stream.log
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: kuiper_logs_with_helm
        path: ./kuiper_logs
    - uses: actions/upload-artifact@v1
      if: always()
      with:
        name: jmeter_logs_with_helm
        path: ./jmeter_logs
    - name: check logs
      run: |
        sudo apt update && sudo apt install -y libxml2-utils
        cd jmeter_logs
        for file in `ls *.jtl`
        do
          if [ ! -z "$(cat $file| grep '<failure>' | awk -F '>' '{print $2}' | awk -F '<' '{print $1}' | grep true)" ] ||
           [ "$(xmllint --format --xpath '/testResults/sample/@rc' $file | sed -r 's/ /\n/g;' | sort -u | grep -E 'rc=\"[45][0-9][0-9]\"|rc=\"\"')" != "" ]; then
            echo -e "---------------------------------------------\n"
            echo "FVT tests error for $file"
            exit 1
          fi
        done