ci: optimize ci workflow

.github/workflows/ci.yml

@@ -0,0 +1,170 @@
+name: EMQX Neuron dev
+
+on: 
+  push:
+  pull_request:
+  release:
+    types:
+      - published
+
+jobs:
+  clang_format_check:
+    runs-on: ubuntu-20.04
+
+    steps:
+      # checkout
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Run clang-format style check for C/C++ programs.
+        uses: DoozyX/clang-format-lint-action@v0.12
+        with:
+          source: 'src plugins include tests'
+          exclude: 'include/vector.h'
+          clangFormatVersion: 10
+          style: file
+
+  cross-build:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        arch:
+          - [aarch64-linux-gnu, aarch64, gcc-aarch64-linux-gnu, g++-aarch64-linux-gnu, /opt/externs/libs]
+          - [arm-linux-gnueabihf, armv4, gcc-arm-linux-gnueabihf, g++-arm-linux-gnueabihf, /opt/externs/libs]
+          - [x86_64-linux-gnu, x86_64, gcc, g++, /usr/local]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      
+      - name: install
+        run: sudo apt-get install -y ${{ matrix.arch[2] }} ${{ matrix.arch[3] }} ninja-build pkg-config
+      
+      - name: install dep
+        run: |
+          ./install.sh -d ${{ matrix.arch[4] }} -c ${{ matrix.arch[0] }} -a ${{ matrix.arch[1] }}
+
+      - name: compile
+        run: |
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/${{ matrix.arch[0] }}.cmake \
+            -DCMAKE_BUILD_TYPE=Release -DDISABLE_UT=ON
+          make -j4
+
+      - name: Create package
+        env:
+          ARCH: ${{ matrix.arch[0] }}
+        run: |
+          ./package.sh -p ${ARCH} -n neuron-${GITHUB_REF##*/}-${ARCH%%-*}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.arch[0] }}
+          path: "*.tar.gz"
+  
+  ut:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      
+      - name: install dep
+        run: |
+          ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu
+
+      - name: compile
+        run: |
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake
+          make -j4
+
+      - name: unit test
+        run: |
+          cd build
+          ctest --output-on-failure
+
+  http_ft:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      
+      - name: install dep
+        run: |
+          ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu
+
+      - name: compile
+        run: |
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake
+          make -j4
+
+      - name: function test
+        run: |
+          python3 -m pip install -U pip
+          python3 -m pip install -r ft/requirements.txt
+          python3 -m robot -P ft/ --variable neuron_api:http -d ft/http_reports ft
+
+  mqtt_ft:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      
+      - name: install dep
+        run: |
+          ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu
+
+      - name: compile
+        run: |
+          mkdir -p build && cd build
+          cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake
+          make -j4
+
+      - name: function test
+        run: |
+          sudo apt-get install -y mosquitto
+          mosquitto -v &
+          python3 -m pip install -U pip
+          python3 -m pip install -r ft/requirements.txt
+          python3 -m robot -P ft/ --variable neuron_api:mqtt -d ft/http_reports ft
+  
+  release:
+    runs-on: ubuntu-20.04
+    if: github.event_name == 'release'
+    needs: cross-build
+    strategy:
+      matrix:
+        arch:
+          - "arm-linux-gnueabihf"
+          - "aarch64-linux-gnu"
+          - "x86_64-linux-gnu"
+
+    steps:
+      - uses: actions/download-artifact@v2
+        id: download
+        with:
+          name: ${{ matrix.arch }}
+
+      - id: package
+        env:
+          ARCH: ${{ matrix.arch }}
+        run: echo "::set-output name=name::neuron-${GITHUB_REF##*/}-${ARCH%%-*}.tar.gz"
+
+      - uses: actions/upload-release-asset@v1
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ./${{ steps.package.outputs.name}}
+          asset_name: ${{ steps.package.outputs.name}}
+          asset_content_type: application/octet-stream

CMakeLists.txt

@@ -56,8 +56,8 @@ if(NOT CMAKE_STAGING_PREFIX)
   endif()
 else()
   set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX})
-  include_directories(${CMAKE_STAGING_PREFIX}/openssl/include)
-  link_directories(${CMAKE_STAGING_PREFIX}/openssl/lib)
+  include_directories(${CMAKE_STAGING_PREFIX}/include ${CMAKE_STAGING_PREFIX}/openssl/include)
+  link_directories(${CMAKE_STAGING_PREFIX} ${CMAKE_STAGING_PREFIX}/lib ${CMAKE_STAGING_PREFIX}/openssl/lib)
 endif()
 
 

cmake/aarch64-linux-gnu.cmake

@@ -1,53 +1,18 @@
 set(CMAKE_SYSTEM_NAME Linux)
+set(COMPILER_PREFIX aarch64-linux-gnu)
 set(CMAKE_SYSTEM_PROCESSOR aarch64)
+set(LIBRARY_DIR /opt/externs/libs)
 
-set(triple aarch64-linux-gnu)
-set(DISABLE_UT ON)
-
-set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple})
-
-if(NOT CMAKE_INSTALL_PREFIX)
-  set(CMAKE_INSTALL_PREFIX .cross/${triple})
-endif()
-
-set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple})
+set(CMAKE_C_COMPILER ${COMPILER_PREFIX}-gcc)
+set(CMAKE_CXX_COMPILER ${COMPILER_PREFIX}-g++)
+set(CMAKE_AR ${COMPILER_PREFIX}-ar)
+set(CMAKE_LINKER ${COMPILER_PREFIX}-ld)
+set(CMAKE_NM ${COMPILER_PREFIX}-nm)
+set(CMAKE_OBJDUMP ${COMPILER_PREFIX}-objdump)
+set(CMAKE_RANLIB ${COMPILER_PREFIX}-ranlib)
+set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX})
 set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX})
 
-set(CMAKE_C_COMPILER ${triple}-gcc)
-set(CMAKE_C_FLAGS
-    "-O3 -fPIC"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_DEBUG
-    "${CMAKE_C_FLAGS} -g -Wall"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_MINSIZEREL
-    "${CMAKE_C_FLAGS} -Os -DNDEBUG"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_RELEASE
-    "${CMAKE_C_FLAGS} -O4 -DNDEBUG"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_RELWITHDEBINFO
-    "${CMAKE_C_FLAGS} -O2 -g"
-    CACHE INTERNAL "")
-
-# set(CMAKE_CXX_COMPILER ${triple}-g++) set(CMAKE_CXX_FLAGS "-O3 -fPIC" CACHE
-# INTERNAL "") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall" CACHE
-# INTERNAL "") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -Os -DNDEBUG"
-# CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O4
-# -DNDEBUG" CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
-# "${CMAKE_CXX_FLAGS} -O2 -g" CACHE INTERNAL "")
-# include_directories(/opt/externs/libs/${triple}/include)
-# include_directories(/opt/externs/libs/${triple}/openssl/include)
-
-# set(CMAKE_C_COMPILER ${triple}-gcc) set(CMAKE_CXX_COMPILER ${triple}-g++)
-
-set(CMAKE_AR ${triple}-ar)
-set(CMAKE_AR ${triple}-ar)
-set(CMAKE_LINKER ${triple}-ld)
-set(CMAKE_NM ${triple}-nm)
-set(CMAKE_OBJDUMP ${triple}-objdump)
-set(CMAKE_RANLIB ${triple}-ranlib)
-
 include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/include)
 include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/openssl/include)
 set(CMAKE_FIND_ROOT_PATH ${CMAKE_STAGING_PREFIX})

cmake/arm-linux-gnueabihf.cmake

@@ -1,52 +1,18 @@
 set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_SYSTEM_VERSION 1)
+set(COMPILER_PREFIX arm-linux-gnueabihf)
 set(CMAKE_SYSTEM_PROCESSOR armv7l)
+set(LIBRARY_DIR /opt/externs/libs)
 
-set(triple arm-linux-gnueabihf)
-set(DISABLE_UT ON)
-
-if(NOT CMAKE_INSTALL_PREFIX)
-  set(CMAKE_INSTALL_PREFIX .cross/${triple})
-endif()
-
-set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple})
+set(CMAKE_C_COMPILER ${COMPILER_PREFIX}-gcc)
+set(CMAKE_CXX_COMPILER ${COMPILER_PREFIX}-g++)
+set(CMAKE_AR ${COMPILER_PREFIX}-ar)
+set(CMAKE_LINKER ${COMPILER_PREFIX}-ld)
+set(CMAKE_NM ${COMPILER_PREFIX}-nm)
+set(CMAKE_OBJDUMP ${COMPILER_PREFIX}-objdump)
+set(CMAKE_RANLIB ${COMPILER_PREFIX}-ranlib)
+set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX})
 set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX})
 
-set(CMAKE_C_COMPILER ${triple}-gcc)
-set(CMAKE_C_FLAGS
-    "-O3 -fPIC"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_DEBUG
-    "${CMAKE_C_FLAGS} -g -Wall"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_MINSIZEREL
-    "${CMAKE_C_FLAGS} -Os -DNDEBUG"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_RELEASE
-    "${CMAKE_C_FLAGS} -O4 -DNDEBUG"
-    CACHE INTERNAL "")
-set(CMAKE_C_FLAGS_RELWITHDEBINFO
-    "${CMAKE_C_FLAGS} -O2 -g"
-    CACHE INTERNAL "")
-
-# set(CMAKE_CXX_COMPILER ${triple}-g++) set(CMAKE_CXX_FLAGS "-O3 -fPIC" CACHE
-# INTERNAL "") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall" CACHE
-# INTERNAL "") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -Os -DNDEBUG"
-# CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O4
-# -DNDEBUG" CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
-# "${CMAKE_CXX_FLAGS} -O2 -g" CACHE INTERNAL "")
-# include_directories(/opt/externs/libs/${triple}/include)
-# include_directories(/opt/externs/libs/${triple}/openssl/include)
-
-# set(CMAKE_C_COMPILER ${triple}-gcc) set(CMAKE_CXX_COMPILER ${triple}-g++)
-
-set(CMAKE_AR ${triple}-ar)
-set(CMAKE_AR ${triple}-ar)
-set(CMAKE_LINKER ${triple}-ld)
-set(CMAKE_NM ${triple}-nm)
-set(CMAKE_OBJDUMP ${triple}-objdump)
-set(CMAKE_RANLIB ${triple}-ranlib)
-
 include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/include)
 include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/openssl/include)
 set(CMAKE_FIND_ROOT_PATH ${CMAKE_STAGING_PREFIX})

cmake/x86_64-linux-gnu.cmake

@@ -1,5 +1,7 @@
 set(CMAKE_SYSTEM_NAME Linux)
-set(CMAKE_SYSTEM_VERSION 1)
+set(COMPILER_PREFIX x86_64-linux-gnu)
 set(CMAKE_SYSTEM_PROCESSOR x86_64)
+set(LIBRARY_DIR /usr/local)
 
-set(triple x86_64-linux-gnu)
+set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX})
+set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX})

install.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -e
+
+compiler_prefix="x86_64-linux-gnu"
+dir="/usr/local/"
+install_dir=
+arch=x86_64
+
+while getopts ":c:d:a:" OPT; do
+    case ${OPT} in
+        a)
+            arch=$OPTARG
+            ;;
+        c)
+            compiler_prefix=$OPTARG
+            ;;
+        d)
+            dir=$OPTARG
+            ;;
+    esac
+done
+
+install_dir=${dir}/${compiler_prefix}
+
+#$1 rep
+#$2 tag
+#$3 compile option
+function compile_source() {
+    git clone https://github.com/$1 tmp
+    cd tmp
+    git checkout -b br $2
+    mkdir build && cd build
+    cmake .. -DCMAKE_C_COMPILER=${compiler_prefix}-gcc \
+        -DCMAKE_CXX_COMPILER=${compiler_prefix}-g++ \
+        -DCMAKE_STAGING_PREFIX=${install_dir} \
+        -DCMAKE_PREFIX_PATH=${install_dir} \
+        $3
+    # github-hosted runners has 2 core
+    make -j4 && sudo make install
+    cd ../../
+    rm -rf tmp
+}
+
+function build_openssl() {
+    echo "Installing openssl (1.1.1)"
+    if [ $compiler_prefix == "x86_64-linux-gnu" ]; then
+        sudo apt-get install -y openssl libssl-dev
+    else
+        git clone -b OpenSSL_1_1_1-stable https://github.com/openssl/openssl.git
+        cd openssl
+        mkdir -p ${install_dir}/openssl/ssl
+        ./Configure linux-${arch} no-asm shared \
+            --prefix=${install_dir}/openssl \
+            --openssldir=${install_dir}/openssl/ssl \
+            --cross-compile-prefix=${compiler_prefix}- \
+            --with-zlib-include=${install_dir}/zlib/include \
+            --with-zlib-lib=${install_dir}/zlib/lib
+        make clean
+        make -j4
+        make install_sw
+        make clean
+        cd ../
+    fi
+}
+
+ssl_lib_flag="-DOPENSSL_ROOT_DIR=${install_dir}/openssl \
+    -DOPENSSL_INCLUDE_DIR=${install_dir}/openssl/include \
+    -DOPENSSL_LIBRARIES='${install_dir}/openssl/lib'"
+
+compile_source yaml/libyaml.git 0.2.5 "-DBUILD_SHARED_LIBS=OFF"
+compile_source madler/zlib.git v1.2.11
+compile_source akheron/jansson.git v2.14 "-DJANSSON_BUILD_DOCS=OFF -DJANSSON_EXAMPLES=OFF"
+build_openssl 
+compile_source neugates/nng.git HEAD "-DBUILD_SHARED_LIBS=OFF -DNNG_TESTS=OFF"
+if [ $compiler_prefix == "x86_64-linux-gnu" ]; then
+    compile_source benmcollins/libjwt.git v1.13.1 "-DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF"
+    compile_source neugates/MQTT-C.git HEAD "-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF"
+    compile_source google/googletest.git release-1.11.0
+else
+    compile_source benmcollins/libjwt.git v1.13.1 "${ssl_lib_flag} -DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF"
+    compile_source neugates/MQTT-C.git HEAD "${ssl_lib_flag} -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF"
+fi

package.sh

@@ -50,8 +50,6 @@ case "$arch" in
     ;;
 esac
 
-cp ${lib_path}/libyaml.so ${package_name}/
-
 cp build/lib*.so \
     build/neuron \
     build/neuron.yaml \