More ci tweak (#2636)

* change concurrency group to ${{ github.workflow }}-${{ github.ref }}
* use argparse for build.py hil_test.py, remove the need to install click
* move ci win/mac to build_cmake.yml
* rename build_family.yml to build_util.yml
* build_util.yml support esp32
* integrate build-espressif into build.yml
* build.py support make with --board option
* add get_deps action
* update hil test to reuse action

.github/actions/get_deps/action.yml

@@ -1,30 +1,29 @@
-name: Prepare to build
+name: Get dependencies
 
 inputs:
-  family:
+  arg:
     required: true
     type: string
 
 runs:
   using: "composite"
   steps:
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-
     - name: Checkout pico-sdk for rp2040
-      if: contains(inputs.family, 'rp2040')
+      if: contains(inputs.arg, 'rp2040') || contains(inputs.arg, 'raspberry_pi_pico')
       uses: actions/checkout@v4
       with:
         repository: raspberrypi/pico-sdk
         ref: develop
         path: pico-sdk
 
-    - name: Get Dependencies
+    - name: Linux dependencies
+      if: runner.os == 'Linux'
       run: |
         sudo apt install -y ninja-build
-        pip install click
-        python3 tools/get_deps.py ${{ inputs.family }}
-        echo >> $GITHUB_ENV "PICO_SDK_PATH=$GITHUB_WORKSPACE/pico-sdk"
+      shell: bash
+
+    - name: Get Dependencies
+      run: |
+        python3 tools/get_deps.py ${{ inputs.arg }}
+        echo "PICO_SDK_PATH=${{ github.workspace }}/pico-sdk" >> $GITHUB_ENV
       shell: bash

.github/actions/setup_toolchain/action.yml

@@ -8,6 +8,11 @@ inputs:
     required: false
     type: string
 
+outputs:
+  build_option:
+    description: 'Build option for the toolchain e.g --toolchain clang'
+    value: ${{ steps.set-toolchain-option.outputs.build_option }}
+
 runs:
   using: "composite"
   steps:
@@ -19,7 +24,7 @@ runs:
 
     - name: Pull ESP-IDF docker
       if: inputs.toolchain == 'esp-idf'
-      run: docker pull espressif/idf:latest
+      run: docker pull espressif/idf:${{ inputs.toolchain_url }}
       shell: bash
 
     - name: Download Toolchain
@@ -29,3 +34,14 @@ runs:
       uses: ./.github/actions/setup_toolchain/download
       with:
         toolchain_url: ${{ inputs.toolchain_url }}
+
+    - name: Set toolchain option
+      id: set-toolchain-option
+      run: |
+          BUILD_OPTION=""
+          if [[ "${{ inputs.toolchain }}" == *"clang"* ]]; then
+            BUILD_OPTION="--toolchain clang"
+          fi
+          echo "build_option=$BUILD_OPTION"
+          echo "build_option=$BUILD_OPTION" >> $GITHUB_OUTPUT
+      shell: bash

.github/workflows/build_cmake.yml

@@ -12,6 +12,7 @@ on:
       - 'tools/build.py'
       - '.github/actions/**'
       - '.github/workflows/build_cmake.yml'
+      - '.github/workflows/build_util.yml'
       - '.github/workflows/ci_set_matrix.py'
   pull_request:
     branches: [ master ]
@@ -24,9 +25,10 @@ on:
       - 'tools/build.py'
       - '.github/actions/**'
       - '.github/workflows/build_cmake.yml'
+      - '.github/workflows/build_util.yml'
       - '.github/workflows/ci_set_matrix.py'
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -55,7 +57,7 @@ jobs:
   # ---------------------------------------
   cmake:
     needs: set-matrix
-    uses: ./.github/workflows/build_family.yml
+    uses: ./.github/workflows/build_util.yml
     strategy:
       fail-fast: false
       matrix:
@@ -69,14 +71,14 @@ jobs:
       build-system: 'cmake'
       toolchain: ${{ matrix.toolchain }}
       toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }}
-      build-family: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
+      build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
 
   # ---------------------------------------
   # Build Make
   # ---------------------------------------
   make:
     needs: set-matrix
-    uses: ./.github/workflows/build_family.yml
+    uses: ./.github/workflows/build_util.yml
     strategy:
       fail-fast: false
       matrix:
@@ -90,4 +92,38 @@ jobs:
       build-system: 'make'
       toolchain: ${{ matrix.toolchain }}
       toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }}
-      build-family: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
+      build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
+
+  # ---------------------------------------
+  # Build Make on Windows/MacOS
+  # ---------------------------------------
+  make-os:
+    uses: ./.github/workflows/build_util.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+    with:
+      os: ${{ matrix.os }}
+      build-system: 'make'
+      toolchain: 'arm-gcc'
+      build-args: '["-bstm32f411disco"]'
+
+  # ---------------------------------------
+  # Build Espressif
+  # ---------------------------------------
+  espressif:
+    uses: ./.github/workflows/build_util.yml
+    strategy:
+      fail-fast: false
+      matrix:
+        board:
+          # ESP32-S2
+          - 'espressif_kaluga_1'
+          # ESP32-S3 skip since devkitm is also compiled in hil-test workflow
+          #- 'espressif_s3_devkitm'
+    with:
+      build-system: 'cmake'
+      toolchain: 'esp-idf'
+      toolchain_url: 'v5.1.1'
+      build-args: '["-b${{ matrix.board }}"]'

.github/workflows/build_esp.yml

@@ -1,109 +0,0 @@
-name: Build ESP
-
-on:
-  workflow_dispatch:
-  push:
-    paths:
-      - 'src/**'
-      - 'examples/**'
-      - 'lib/**'
-      - 'hw/**'
-      - 'test/hil/**'
-      - '.github/workflows/build_esp.yml'
-  pull_request:
-    branches: [ master ]
-    paths:
-      - 'src/**'
-      - 'examples/**'
-      - 'lib/**'
-      - 'hw/**'
-      - 'test/hil/**'
-      - '.github/workflows/build_esp.yml'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  build-esp:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        board:
-        # ESP32-S2
-        - 'espressif_kaluga_1'
-        # ESP32-S3
-        - 'espressif_s3_devkitm'
-    steps:
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-
-    - name: Checkout TinyUSB
-      uses: actions/checkout@v4
-
-    - name: Setup Toolchain
-      uses: ./.github/actions/setup_toolchain
-      with:
-        toolchain: 'esp-idf'
-
-    - name: Build
-      run: docker run --rm -v $PWD:/project -w /project espressif/idf:v5.1.1 python3 tools/build.py -b ${{ matrix.board }}
-
-    - name: Upload Artifacts for Hardware Testing
-      if: matrix.board == 'espressif_s3_devkitm' && github.repository_owner == 'hathach'
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ matrix.board }}
-        path: |
-          cmake-build/cmake-build-${{ matrix.board }}/*/*/bootloader/bootloader.bin
-          cmake-build/cmake-build-${{ matrix.board }}/*/*/*.bin
-          cmake-build/cmake-build-${{ matrix.board }}/*/*/partition_table/partition-table.bin
-          cmake-build/cmake-build-${{ matrix.board }}/*/*/config.env
-          cmake-build/cmake-build-${{ matrix.board }}/*/*/flash_args
-
-  # ---------------------------------------
-  # Hardware in the loop (HIL)
-  # Current self-hosted instance is running on an RPI4. For attached hardware checkout hil_pi4.json
-  # ---------------------------------------
-  hil-test:
-    # run only with hathach's commit due to limited resource on RPI4
-    if: github.repository_owner == 'hathach'
-    needs: build-esp
-    runs-on: [self-hosted, esp32s3, hardware-in-the-loop]
-    strategy:
-      fail-fast: false
-      matrix:
-        board:
-          - 'espressif_s3_devkitm'
-    steps:
-      - name: Clean workspace
-        run: |
-          echo "Cleaning up previous run"
-          rm -rf "${{ github.workspace }}"
-          mkdir -p "${{ github.workspace }}"
-
-        # USB bus on rpi4 is not stable, reset it before testing
-      - name: Reset USB bus
-        run: |
-          lsusb
-          lsusb -t
-          # reset VIA Labs 2.0 hub
-          sudo usbreset 001/002
-
-      - name: Checkout test/hil
-        uses: actions/checkout@v4
-        with:
-          sparse-checkout: test/hil
-
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ matrix.board }}
-          path: cmake-build/cmake-build-${{ matrix.board }}
-
-      - name: Test on actual hardware
-        run: |
-          python3 test/hil/hil_test.py --board ${{ matrix.board }} pi4_esp32.json

.github/workflows/build_family.yml

@@ -1,64 +0,0 @@
-name: Build family
-
-on:
-  workflow_call:
-    inputs:
-      build-system:
-        required: true
-        type: string
-      toolchain:
-        required: true
-        type: string
-      toolchain_url:
-        required: true
-        type: string
-      build-family:
-        required: true
-        type: string
-
-jobs:
-  family:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        family: ${{ fromJSON(inputs.build-family) }}
-    steps:
-      - name: Checkout TinyUSB
-        uses: actions/checkout@v4
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-
-      - name: Setup Toolchain
-        uses: ./.github/actions/setup_toolchain
-        with:
-          toolchain: ${{ inputs.toolchain }}
-          toolchain_url: ${{ inputs.toolchain_url }}
-
-      - name: Checkout pico-sdk for rp2040
-        if: contains(matrix.family, 'rp2040')
-        uses: actions/checkout@v4
-        with:
-          repository: raspberrypi/pico-sdk
-          ref: develop
-          path: pico-sdk
-
-      - name: Get Dependencies
-        run: |
-          sudo apt install -y ninja-build
-          pip install click
-          python3 tools/get_deps.py ${{ matrix.family }}
-
-      - name: Build
-        run: |
-          OPTION=""
-          if [[ "${{ inputs.toolchain }}" == *"clang"* ]]; then
-            OPTION="--toolchain clang"
-          fi
-          echo "OPTION=$OPTION"
-          python tools/build.py -s ${{ inputs.build-system }} $OPTION ${{ matrix.family }}
-        env:
-          PICO_SDK_PATH: ${{ github.workspace }}/pico-sdk

.github/workflows/build_iar.yml

@@ -23,7 +23,7 @@ on:
       - '.github/workflows/build_iar.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:

.github/workflows/build_renesas.yml

@@ -21,7 +21,7 @@ on:
       - '.github/workflows/build_renesas.yml'
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -65,7 +65,6 @@ jobs:
 
     - name: Get Dependencies
       run: |
-        pip install click
         python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build

.github/workflows/build_util.yml

@@ -0,0 +1,59 @@
+name: Reusable build util
+
+on:
+  workflow_call:
+    inputs:
+      build-system:
+        required: true
+        type: string
+      toolchain:
+        required: true
+        type: string
+      toolchain_url:
+        required: false
+        type: string
+      build-args:
+        required: true
+        type: string
+      os:
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+
+jobs:
+  family:
+    runs-on: ${{ inputs.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arg: ${{ fromJSON(inputs.build-args) }}
+    steps:
+      - name: Checkout TinyUSB
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Setup Toolchain
+        id: setup-toolchain
+        uses: ./.github/actions/setup_toolchain
+        with:
+          toolchain: ${{ inputs.toolchain }}
+          toolchain_url: ${{ inputs.toolchain_url }}
+
+      - name: Get Dependencies
+        uses: ./.github/actions/get_deps
+        with:
+          arg: ${{ matrix.arg }}
+
+      - name: Build
+        if: inputs.toolchain != 'esp-idf'
+        run: |
+          python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ matrix.arg }}
+
+      - name: Build using ESP-IDF docker
+        if: inputs.toolchain == 'esp-idf'
+        run: |
+          docker run --rm -v $PWD:/project -w /project espressif/idf:${{ inputs.toolchain_url }} python3 tools/build.py ${{ matrix.arg }}

.github/workflows/build_win_mac.yml

@@ -1,56 +0,0 @@
-name: Build Windows/MacOS
-
-on:
-  workflow_dispatch:
-  push:
-    paths:
-      - 'src/**'
-      - 'examples/**'
-      - 'lib/**'
-      - 'hw/**'
-      - '.github/workflows/build_win_mac.yml'
-  pull_request:
-    branches: [ master ]
-    paths:
-      - 'src/**'
-      - 'examples/**'
-      - 'lib/**'
-      - 'hw/**'
-      - '.github/workflows/build_win_mac.yml'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  # ---------------------------------------
-  # Build ARM family
-  # ---------------------------------------
-  build-arm:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.x'
-
-    - name: Install ARM GCC
-      uses: carlosperate/arm-none-eabi-gcc-action@v1
-      with:
-        release: '10.3-2021.10'
-
-    - name: Checkout TinyUSB
-      uses: actions/checkout@v4
-
-    - name: Get Dependencies
-      run: |
-        pip install click
-        python3 tools/get_deps.py stm32f4
-
-    - name: Build
-      run: python3 tools/build.py -s make stm32f2

.github/workflows/ci_set_matrix.py

@@ -16,7 +16,7 @@ family_list = {
     "ch32v307 fomu gd32vf103": ["riscv-gcc"],
     "imxrt": ["arm-gcc", "arm-clang"],
     "kinetis_k kinetis_kl kinetis_k32l2": ["arm-gcc", "arm-clang"],
-    "lpc11 lpc13 lpc15": ["arm-gcc"],
+    "lpc11 lpc13 lpc15": ["arm-gcc", "arm-clang"],
     "lpc17 lpc18 lpc40 lpc43": ["arm-gcc", "arm-clang"],
     "lpc51 lpc54 lpc55": ["arm-gcc", "arm-clang"],
     "mcx": ["arm-gcc"],

.github/workflows/hil_test.yml

@@ -24,15 +24,18 @@ on:
       - '.github/actions/**'
       - '.github/workflows/hil_test.yml'
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  # ---------------------------------------
+  # Build Non Espressif
+  # ---------------------------------------
   build:
     if: github.repository_owner == 'hathach'
     runs-on: ubuntu-latest
     outputs:
-      BOARD_LIST: ${{ steps.parse_hil_json.outputs.BOARD_LIST }}
+      BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }}
     steps:
       - name: Checkout TinyUSB
         uses: actions/checkout@v4
@@ -42,38 +45,29 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Install ARM GCC
-        uses: carlosperate/arm-none-eabi-gcc-action@v1
-        with:
-          release: '12.3.Rel1'
-
       - name: Parse HIL json
         id: parse_hil_json
         run: |
           sudo apt install -y jq
-          BOARD_LIST=$(jq -r '.boards[] | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
-          echo "BOARD_LIST=$BOARD_LIST"
-          echo >> $GITHUB_ENV "BOARD_LIST=$BOARD_LIST"
-          echo >> $GITHUB_OUTPUT "BOARD_LIST=$BOARD_LIST"
 
-      - name: Checkout pico-sdk for rp2040
-        uses: actions/checkout@v4
+          # Non-Espresif boards
+          BOARDS_LIST=$(jq -r '.boards[] | select(.flasher != "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT
+
+      - name: Setup ARM Toolchain
+        uses: ./.github/actions/setup_toolchain
         with:
-          repository: raspberrypi/pico-sdk
-          ref: develop
-          path: pico-sdk
+          toolchain: 'arm-gcc'
 
       - name: Get Dependencies
-        run: |
-          pip install click
-          sudo apt install -y ninja-build
-          python3 tools/get_deps.py $BOARD_LIST
+        uses: ./.github/actions/get_deps
+        with:
+          arg: ${{ env.BOARDS_LIST }}
 
       - name: Build
-        run: |
-          python tools/build.py $BOARD_LIST
-        env:
-          PICO_SDK_PATH: ${{ github.workspace }}/pico-sdk
+        run: python tools/build.py $BOARDS_LIST
 
       - name: Upload Artifacts for Hardware Testing
         uses: actions/upload-artifact@v4
@@ -83,16 +77,71 @@ jobs:
             cmake-build/cmake-build-*/*/*/*.elf
             cmake-build/cmake-build-*/*/*/*.bin
 
+  # ---------------------------------------
+  # Build Espressif
+  # ---------------------------------------
+  build-esp:
+    runs-on: ubuntu-latest
+    outputs:
+      BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }}
+    steps:
+      - name: Checkout TinyUSB
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Parse HIL json
+        id: parse_hil_json
+        run: |
+          sudo apt install -y jq
+          # Espressif boards
+          BOARDS_LIST=$(jq -r '.boards[] | select(.flasher == "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT
+
+      - name: Setup ESP-IDF
+        if: env.BOARDS_LIST != ''
+        uses: ./.github/actions/setup_toolchain
+        with:
+          toolchain: 'esp-idf'
+          toolchain_url: 'v5.1.1'
+
+      - name: Get Dependencies
+        uses: ./.github/actions/get_deps
+        with:
+          arg: ${{ env.BOARDS_LIST }}
+
+      - name: Build Espressif
+        if: env.BOARDS_LIST != ''
+        run: docker run --rm -v $PWD:/project -w /project espressif/idf:v5.1.1 python3 tools/build.py $BOARDS_LIST
+
+      - name: Upload Artifacts for Hardware Testing
+        uses: actions/upload-artifact@v4
+        with:
+          name: hil_pi4_esp
+          path: |
+            cmake-build/cmake-build-*/*/*/*.bin
+            cmake-build/cmake-build-*/*/*/bootloader/bootloader.bin
+            cmake-build/cmake-build-*/*/*/partition_table/partition-table.bin
+            cmake-build/cmake-build-*/*/*/config.env
+            cmake-build/cmake-build-*/*/*/flash_args
+
   # ---------------------------------------
   # Hardware in the loop (HIL)
-  # Current self-hosted instance is running on an RPI4. For attached hardware checkout hil_pi4.json
+  # Current self-hosted instance is running on an RPI4. For attached hardware checkout test/hil/pi4.json
   # ---------------------------------------
   hil-pi4:
     if: github.repository_owner == 'hathach'
-    needs: build
-    runs-on: [self-hosted, rp2040, nrf52840, hardware-in-the-loop]
+    needs:
+      - build
+      - build-esp
+    runs-on: [self-hosted, rp2040, nrf52840, esp32s3, hardware-in-the-loop]
     env:
-        BOARD_LIST: ${{ needs.build.outputs.BOARD_LIST }}
+        BOARDS_LIST: "${{ needs.build.outputs.BOARDS_LIST }} ${{ needs.build-esp.outputs.BOARDS_LIST }}"
     steps:
       - name: Clean workspace
         run: |
@@ -103,8 +152,7 @@ jobs:
         # USB bus on rpi4 is not stable, reset it before testing
       - name: Reset USB bus
         run: |
-          lsusb
-          lsusb -t
+          # lsusb -t
           # reset VIA Labs 2.0 hub
           sudo usbreset 001/002
 
@@ -119,7 +167,16 @@ jobs:
           name: hil_pi4
           path: cmake-build
 
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: hil_pi4_esp
+          path: cmake-build
+
       - name: Test on actual hardware
         run: |
-          echo "BOARD_LIST=$BOARD_LIST"
-          python3 test/hil/hil_test.py $BOARD_LIST pi4.json
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "::group::{cmake-build contents}"
+          tree cmake-build
+          echo "::endgroup::"
+          python3 test/hil/hil_test.py $BOARDS_LIST pi4.json

.github/workflows/pre-commit.yml

@@ -7,7 +7,7 @@ on:
     branches: [ master ]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
@@ -38,7 +38,6 @@ jobs:
 
     - name: Build Fuzzer
       run: |
-        pip install click
         export CC=clang
         export CXX=clang++
         fuzz_harness=$(ls -d test/fuzz/device/*/)