From 6bbfcd47a93b5041f2bb25daa07bf4fa928f6b12 Mon Sep 17 00:00:00 2001 From: fengzero Date: Mon, 7 Mar 2022 03:19:56 +0000 Subject: [PATCH] ci: optimize ci workflow --- .github/workflows/ci.yml | 170 ++++++++++++++++++++++++++++++++ CMakeLists.txt | 4 +- cmake/aarch64-linux-gnu.cmake | 55 ++--------- cmake/arm-linux-gnueabihf.cmake | 54 ++-------- cmake/x86_64-linux-gnu.cmake | 6 +- install.sh | 83 ++++++++++++++++ package.sh | 2 - 7 files changed, 279 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100755 install.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..2a916ca6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,170 @@ +name: EMQX Neuron dev + +on: + push: + pull_request: + release: + types: + - published + +jobs: + clang_format_check: + runs-on: ubuntu-20.04 + + steps: + # checkout + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Run clang-format style check for C/C++ programs. + uses: DoozyX/clang-format-lint-action@v0.12 + with: + source: 'src plugins include tests' + exclude: 'include/vector.h' + clangFormatVersion: 10 + style: file + + cross-build: + runs-on: ubuntu-20.04 + strategy: + matrix: + arch: + - [aarch64-linux-gnu, aarch64, gcc-aarch64-linux-gnu, g++-aarch64-linux-gnu, /opt/externs/libs] + - [arm-linux-gnueabihf, armv4, gcc-arm-linux-gnueabihf, g++-arm-linux-gnueabihf, /opt/externs/libs] + - [x86_64-linux-gnu, x86_64, gcc, g++, /usr/local] + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: install + run: sudo apt-get install -y ${{ matrix.arch[2] }} ${{ matrix.arch[3] }} ninja-build pkg-config + + - name: install dep + run: | + ./install.sh -d ${{ matrix.arch[4] }} -c ${{ matrix.arch[0] }} -a ${{ matrix.arch[1] }} + + - name: compile + run: | + mkdir -p build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/${{ matrix.arch[0] }}.cmake \ + -DCMAKE_BUILD_TYPE=Release -DDISABLE_UT=ON + make -j4 + + - name: Create package + env: + ARCH: ${{ matrix.arch[0] }} + run: | + ./package.sh -p ${ARCH} -n neuron-${GITHUB_REF##*/}-${ARCH%%-*} + + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.arch[0] }} + path: "*.tar.gz" + + ut: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: install dep + run: | + ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu + + - name: compile + run: | + mkdir -p build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake + make -j4 + + - name: unit test + run: | + cd build + ctest --output-on-failure + + http_ft: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: install dep + run: | + ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu + + - name: compile + run: | + mkdir -p build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake + make -j4 + + - name: function test + run: | + python3 -m pip install -U pip + python3 -m pip install -r ft/requirements.txt + python3 -m robot -P ft/ --variable neuron_api:http -d ft/http_reports ft + + mqtt_ft: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: install dep + run: | + ./install.sh -d /usr/local -a x86_64 -c x86_64-linux-gnu + + - name: compile + run: | + mkdir -p build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=../cmake/x86_64-linux-gnu.cmake + make -j4 + + - name: function test + run: | + sudo apt-get install -y mosquitto + mosquitto -v & + python3 -m pip install -U pip + python3 -m pip install -r ft/requirements.txt + python3 -m robot -P ft/ --variable neuron_api:mqtt -d ft/http_reports ft + + release: + runs-on: ubuntu-20.04 + if: github.event_name == 'release' + needs: cross-build + strategy: + matrix: + arch: + - "arm-linux-gnueabihf" + - "aarch64-linux-gnu" + - "x86_64-linux-gnu" + + steps: + - uses: actions/download-artifact@v2 + id: download + with: + name: ${{ matrix.arch }} + + - id: package + env: + ARCH: ${{ matrix.arch }} + run: echo "::set-output name=name::neuron-${GITHUB_REF##*/}-${ARCH%%-*}.tar.gz" + + - uses: actions/upload-release-asset@v1 + if: github.event_name == 'release' + env: + GITHUB_TOKEN: ${{ github.token }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./${{ steps.package.outputs.name}} + asset_name: ${{ steps.package.outputs.name}} + asset_content_type: application/octet-stream diff --git a/CMakeLists.txt b/CMakeLists.txt index 393b4185..22baa2a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,8 +56,8 @@ if(NOT CMAKE_STAGING_PREFIX) endif() else() set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX}) - include_directories(${CMAKE_STAGING_PREFIX}/openssl/include) - link_directories(${CMAKE_STAGING_PREFIX}/openssl/lib) + include_directories(${CMAKE_STAGING_PREFIX}/include ${CMAKE_STAGING_PREFIX}/openssl/include) + link_directories(${CMAKE_STAGING_PREFIX} ${CMAKE_STAGING_PREFIX}/lib ${CMAKE_STAGING_PREFIX}/openssl/lib) endif() diff --git a/cmake/aarch64-linux-gnu.cmake b/cmake/aarch64-linux-gnu.cmake index 0a481a15..dd35055d 100644 --- a/cmake/aarch64-linux-gnu.cmake +++ b/cmake/aarch64-linux-gnu.cmake @@ -1,53 +1,18 @@ set(CMAKE_SYSTEM_NAME Linux) +set(COMPILER_PREFIX aarch64-linux-gnu) set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(LIBRARY_DIR /opt/externs/libs) -set(triple aarch64-linux-gnu) -set(DISABLE_UT ON) - -set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple}) - -if(NOT CMAKE_INSTALL_PREFIX) - set(CMAKE_INSTALL_PREFIX .cross/${triple}) -endif() - -set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple}) +set(CMAKE_C_COMPILER ${COMPILER_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${COMPILER_PREFIX}-g++) +set(CMAKE_AR ${COMPILER_PREFIX}-ar) +set(CMAKE_LINKER ${COMPILER_PREFIX}-ld) +set(CMAKE_NM ${COMPILER_PREFIX}-nm) +set(CMAKE_OBJDUMP ${COMPILER_PREFIX}-objdump) +set(CMAKE_RANLIB ${COMPILER_PREFIX}-ranlib) +set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX}) set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX}) -set(CMAKE_C_COMPILER ${triple}-gcc) -set(CMAKE_C_FLAGS - "-O3 -fPIC" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_DEBUG - "${CMAKE_C_FLAGS} -g -Wall" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_MINSIZEREL - "${CMAKE_C_FLAGS} -Os -DNDEBUG" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_RELEASE - "${CMAKE_C_FLAGS} -O4 -DNDEBUG" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_RELWITHDEBINFO - "${CMAKE_C_FLAGS} -O2 -g" - CACHE INTERNAL "") - -# set(CMAKE_CXX_COMPILER ${triple}-g++) set(CMAKE_CXX_FLAGS "-O3 -fPIC" CACHE -# INTERNAL "") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall" CACHE -# INTERNAL "") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -Os -DNDEBUG" -# CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O4 -# -DNDEBUG" CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO -# "${CMAKE_CXX_FLAGS} -O2 -g" CACHE INTERNAL "") -# include_directories(/opt/externs/libs/${triple}/include) -# include_directories(/opt/externs/libs/${triple}/openssl/include) - -# set(CMAKE_C_COMPILER ${triple}-gcc) set(CMAKE_CXX_COMPILER ${triple}-g++) - -set(CMAKE_AR ${triple}-ar) -set(CMAKE_AR ${triple}-ar) -set(CMAKE_LINKER ${triple}-ld) -set(CMAKE_NM ${triple}-nm) -set(CMAKE_OBJDUMP ${triple}-objdump) -set(CMAKE_RANLIB ${triple}-ranlib) - include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/include) include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/openssl/include) set(CMAKE_FIND_ROOT_PATH ${CMAKE_STAGING_PREFIX}) diff --git a/cmake/arm-linux-gnueabihf.cmake b/cmake/arm-linux-gnueabihf.cmake index f3612b73..3decc4f0 100644 --- a/cmake/arm-linux-gnueabihf.cmake +++ b/cmake/arm-linux-gnueabihf.cmake @@ -1,52 +1,18 @@ set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) +set(COMPILER_PREFIX arm-linux-gnueabihf) set(CMAKE_SYSTEM_PROCESSOR armv7l) +set(LIBRARY_DIR /opt/externs/libs) -set(triple arm-linux-gnueabihf) -set(DISABLE_UT ON) - -if(NOT CMAKE_INSTALL_PREFIX) - set(CMAKE_INSTALL_PREFIX .cross/${triple}) -endif() - -set(CMAKE_STAGING_PREFIX /opt/externs/libs/${triple}) +set(CMAKE_C_COMPILER ${COMPILER_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${COMPILER_PREFIX}-g++) +set(CMAKE_AR ${COMPILER_PREFIX}-ar) +set(CMAKE_LINKER ${COMPILER_PREFIX}-ld) +set(CMAKE_NM ${COMPILER_PREFIX}-nm) +set(CMAKE_OBJDUMP ${COMPILER_PREFIX}-objdump) +set(CMAKE_RANLIB ${COMPILER_PREFIX}-ranlib) +set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX}) set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX}) -set(CMAKE_C_COMPILER ${triple}-gcc) -set(CMAKE_C_FLAGS - "-O3 -fPIC" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_DEBUG - "${CMAKE_C_FLAGS} -g -Wall" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_MINSIZEREL - "${CMAKE_C_FLAGS} -Os -DNDEBUG" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_RELEASE - "${CMAKE_C_FLAGS} -O4 -DNDEBUG" - CACHE INTERNAL "") -set(CMAKE_C_FLAGS_RELWITHDEBINFO - "${CMAKE_C_FLAGS} -O2 -g" - CACHE INTERNAL "") - -# set(CMAKE_CXX_COMPILER ${triple}-g++) set(CMAKE_CXX_FLAGS "-O3 -fPIC" CACHE -# INTERNAL "") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g -Wall" CACHE -# INTERNAL "") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS} -Os -DNDEBUG" -# CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} -O4 -# -DNDEBUG" CACHE INTERNAL "") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO -# "${CMAKE_CXX_FLAGS} -O2 -g" CACHE INTERNAL "") -# include_directories(/opt/externs/libs/${triple}/include) -# include_directories(/opt/externs/libs/${triple}/openssl/include) - -# set(CMAKE_C_COMPILER ${triple}-gcc) set(CMAKE_CXX_COMPILER ${triple}-g++) - -set(CMAKE_AR ${triple}-ar) -set(CMAKE_AR ${triple}-ar) -set(CMAKE_LINKER ${triple}-ld) -set(CMAKE_NM ${triple}-nm) -set(CMAKE_OBJDUMP ${triple}-objdump) -set(CMAKE_RANLIB ${triple}-ranlib) - include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/include) include_directories(SYSTEM ${CMAKE_STAGING_PREFIX}/openssl/include) set(CMAKE_FIND_ROOT_PATH ${CMAKE_STAGING_PREFIX}) diff --git a/cmake/x86_64-linux-gnu.cmake b/cmake/x86_64-linux-gnu.cmake index 07f670b2..abd0082b 100644 --- a/cmake/x86_64-linux-gnu.cmake +++ b/cmake/x86_64-linux-gnu.cmake @@ -1,5 +1,7 @@ set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) +set(COMPILER_PREFIX x86_64-linux-gnu) set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(LIBRARY_DIR /usr/local) -set(triple x86_64-linux-gnu) +set(CMAKE_STAGING_PREFIX ${LIBRARY_DIR}/${COMPILER_PREFIX}) +set(CMAKE_PREFIX_PATH ${CMAKE_STAGING_PREFIX}) \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100755 index 00000000..ebec325f --- /dev/null +++ b/install.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -e + +compiler_prefix="x86_64-linux-gnu" +dir="/usr/local/" +install_dir= +arch=x86_64 + +while getopts ":c:d:a:" OPT; do + case ${OPT} in + a) + arch=$OPTARG + ;; + c) + compiler_prefix=$OPTARG + ;; + d) + dir=$OPTARG + ;; + esac +done + +install_dir=${dir}/${compiler_prefix} + +#$1 rep +#$2 tag +#$3 compile option +function compile_source() { + git clone https://github.com/$1 tmp + cd tmp + git checkout -b br $2 + mkdir build && cd build + cmake .. -DCMAKE_C_COMPILER=${compiler_prefix}-gcc \ + -DCMAKE_CXX_COMPILER=${compiler_prefix}-g++ \ + -DCMAKE_STAGING_PREFIX=${install_dir} \ + -DCMAKE_PREFIX_PATH=${install_dir} \ + $3 + # github-hosted runners has 2 core + make -j4 && sudo make install + cd ../../ + rm -rf tmp +} + +function build_openssl() { + echo "Installing openssl (1.1.1)" + if [ $compiler_prefix == "x86_64-linux-gnu" ]; then + sudo apt-get install -y openssl libssl-dev + else + git clone -b OpenSSL_1_1_1-stable https://github.com/openssl/openssl.git + cd openssl + mkdir -p ${install_dir}/openssl/ssl + ./Configure linux-${arch} no-asm shared \ + --prefix=${install_dir}/openssl \ + --openssldir=${install_dir}/openssl/ssl \ + --cross-compile-prefix=${compiler_prefix}- \ + --with-zlib-include=${install_dir}/zlib/include \ + --with-zlib-lib=${install_dir}/zlib/lib + make clean + make -j4 + make install_sw + make clean + cd ../ + fi +} + +ssl_lib_flag="-DOPENSSL_ROOT_DIR=${install_dir}/openssl \ + -DOPENSSL_INCLUDE_DIR=${install_dir}/openssl/include \ + -DOPENSSL_LIBRARIES='${install_dir}/openssl/lib'" + +compile_source yaml/libyaml.git 0.2.5 "-DBUILD_SHARED_LIBS=OFF" +compile_source madler/zlib.git v1.2.11 +compile_source akheron/jansson.git v2.14 "-DJANSSON_BUILD_DOCS=OFF -DJANSSON_EXAMPLES=OFF" +build_openssl +compile_source neugates/nng.git HEAD "-DBUILD_SHARED_LIBS=OFF -DNNG_TESTS=OFF" +if [ $compiler_prefix == "x86_64-linux-gnu" ]; then + compile_source benmcollins/libjwt.git v1.13.1 "-DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF" + compile_source neugates/MQTT-C.git HEAD "-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF" + compile_source google/googletest.git release-1.11.0 +else + compile_source benmcollins/libjwt.git v1.13.1 "${ssl_lib_flag} -DENABLE_PIC=ON -DBUILD_SHARED_LIBS=OFF" + compile_source neugates/MQTT-C.git HEAD "${ssl_lib_flag} -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DMQTT_C_OpenSSL_SUPPORT=ON -DMQTT_C_EXAMPLES=OFF" +fi diff --git a/package.sh b/package.sh index c2d8ddc9..97cd7330 100755 --- a/package.sh +++ b/package.sh @@ -50,8 +50,6 @@ case "$arch" in ;; esac -cp ${lib_path}/libyaml.so ${package_name}/ - cp build/lib*.so \ build/neuron \ build/neuron.yaml \