From 0c197a2eae3e7443e115809b3d2d58cf74cc92c5 Mon Sep 17 00:00:00 2001 From: hathach Date: Tue, 8 Jul 2025 16:39:12 +0700 Subject: [PATCH 1/2] try to skip passed board in hil test when re-run --- .github/workflows/build.yml | 11 ++++++++++- test/hil/hil_test.py | 18 +++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e11f8a29..93c48b2b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -200,6 +200,15 @@ jobs: steps: - name: Clean workspace run: | + # Skip boards that passed with previous run + if [ -f ${{ env.HIL_JSON }}.skip ]; then + SKIP_ARGS=$(cat "${HIL_JSON}.skip") + else + SKIP_ARGS="" + fi + echo "SKIP_ARGS=$SKIP_ARGS" + echo "SKIP_ARGS=$SKIP_ARGS" >> $GITHUB_ENV + echo "Cleaning up previous run" rm -rf "${{ github.workspace }}" mkdir -p "${{ github.workspace }}" @@ -218,7 +227,7 @@ jobs: - name: Test on actual hardware run: | ls cmake-build/ - python3 test/hil/hil_test.py ${{ env.HIL_JSON }} + python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_ARGS # --------------------------------------- # Hardware in the loop (HIL) diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index f292bca15..5646d9a26 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -606,7 +606,7 @@ def test_board(board): # flash board_test last to disable board's usb test_example(board, flags_on_list[0], 'device/board_test') - return err_count + return name, err_count def main(): @@ -620,11 +620,13 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('config_file', help='Configuration JSON file') parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified') + parser.add_argument('-s', '--skip', action='append', default=[], help='Skip boards from test') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() config_file = args.config_file boards = args.board + skip_boards = args.skip verbose = args.verbose # if config file is not found, try to find it in the same directory as this script @@ -634,12 +636,22 @@ def main(): config = json.load(f) if len(boards) == 0: - config_boards = config['boards'] + config_boards = [e for e in config['boards'] if e['name'] not in skip_boards] else: config_boards = [e for e in config['boards'] if e['name'] in boards] + err_count = 0 with Pool(processes=os.cpu_count()) as pool: - err_count = sum(pool.map(test_board, config_boards)) + mret = pool.map(test_board, config_boards) + err_count = sum(e[1] for e in mret) + # generate skip list for next re-run if failed + skip_fname = f'{config_file}.skip' + if err_count > 0: + skip_boards += [name for name, err in mret if err == 0] + with open(skip_fname, 'w') as f: + f.write(' '.join(f'-s {i}' for i in skip_boards)) + elif os.path.exists(skip_fname): + os.remove(skip_fname) duration = time.time() - duration print() From 091c6a7889d8973c06415ff13c118c9e32516cf7 Mon Sep 17 00:00:00 2001 From: hathach Date: Tue, 8 Jul 2025 17:20:48 +0700 Subject: [PATCH 2/2] only clean/checkout/download artifacts on first attempt --- .github/workflows/build.yml | 24 +++++++++++++----------- test/hil/hil_test.py | 21 +++++++++++---------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 93c48b2b1..ff59421ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,26 +199,20 @@ jobs: runs-on: [self-hosted, X64, hathach, hardware-in-the-loop] steps: - name: Clean workspace + if: github.run_attempt == '1' run: | - # Skip boards that passed with previous run - if [ -f ${{ env.HIL_JSON }}.skip ]; then - SKIP_ARGS=$(cat "${HIL_JSON}.skip") - else - SKIP_ARGS="" - fi - echo "SKIP_ARGS=$SKIP_ARGS" - echo "SKIP_ARGS=$SKIP_ARGS" >> $GITHUB_ENV - - echo "Cleaning up previous run" + echo "Cleaning up for the first run" rm -rf "${{ github.workspace }}" mkdir -p "${{ github.workspace }}" - name: Checkout TinyUSB + if: github.run_attempt == '1' uses: actions/checkout@v4 with: sparse-checkout: test/hil - name: Download Artifacts + if: github.run_attempt == '1' uses: actions/download-artifact@v4 with: path: cmake-build @@ -227,7 +221,15 @@ jobs: - name: Test on actual hardware run: | ls cmake-build/ - python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_ARGS + + # Skip boards that passed with previous run, file is generated by hil_test.py + SKIP_BOARDS="" + if [ -f ${{ env.HIL_JSON }}.skip ]; then + SKIP_BOARDS=$(cat "${HIL_JSON}.skip") + fi + echo "SKIP_BOARDS=$SKIP_BOARDS" + + python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_BOARDS # --------------------------------------- # Hardware in the loop (HIL) diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 5646d9a26..93cc0a750 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -533,7 +533,7 @@ def test_example(board, f1, example): if not os.path.exists(fw_dir): fw_dir = f'{TINYUSB_ROOT}/examples/cmake-build-{name}{f1_str}/{example}' fw_name = f'{fw_dir}/{os.path.basename(example)}' - print(f'{name+f1_str:40} {example:30} ... ', end='') + print(f'{name+f1_str:40} {example:30} ...', end='') if not os.path.exists(fw_dir) or not (os.path.exists(f'{fw_name}.elf') or os.path.exists(f'{fw_name}.bin')): print('Skip (no binary)') @@ -544,29 +544,30 @@ def test_example(board, f1, example): # flash firmware. It may fail randomly, retry a few times max_rety = 3 + start_s = time.time() for i in range(max_rety): ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, fw_name) if ret.returncode == 0: try: globals()[f'test_{example.replace("/", "_")}'](board) - print('OK') + print(' OK', end='') break except Exception as e: if i == max_rety - 1: err_count += 1 - print(STATUS_FAILED) - print(f' {e}') + print(f'{STATUS_FAILED}: {e}') else: - print() - print(f' Test failed: {e}, retry {i+2}/{max_rety}') - time.sleep(1) + print(f'\n Test failed: {e}, retry {i+2}/{max_rety}', end='') + time.sleep(0.5) else: - print(f'Flashing failed, retry {i+2}/{max_rety}') - time.sleep(1) + print(f'\n Flash failed, retry {i+2}/{max_rety}', end='') + time.sleep(0.5) if ret.returncode != 0: err_count += 1 - print(f'Flash {STATUS_FAILED}') + print(f' Flash {STATUS_FAILED}', end='') + + print(f' in {time.time() - start_s:.1f}s') return err_count