Circi dynamic config (#2763)

Circleci
* build cmake armgcc and arm clang on circleci
* use docker medium+

.circleci/config.yml

@@ -1,92 +1,39 @@
 version: 2.1
 
-commands:
+setup: true
-  setup-toolchain:
+orbs:
-    parameters:
+  continuation: circleci/continuation@1
-        toolchain:
-            type: string
-        toolchain_url:
-            type: string
-    steps:
-#      - run:
-#          name: Make toolchain cache key
-#          command: echo "<< parameters.toolchain >>-<< parameters.toolchain_url>>" > toolchain_key
-#      - restore_cache:
-#          name: Restore Toolchain Cache
-#          key: deps-{{ checksum "toolchain_key" }}
-#          paths:
-#            - ~/cache/<< parameters.toolchain >>
-      - run:
-          name: Install Toolchain
-          command: |
-            # Only download if folder does not exist (not cached)
-            if [ ! -d ~/cache/<< parameters.toolchain >> ]; then
-              mkdir -p ~/cache/<< parameters.toolchain >>
-              wget << parameters.toolchain_url>> -O toolchain.tar.gz
-              tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz
-            fi
-#      - save_cache:
-#          name: Save Toolchain Cache
-#          key: deps-{{ checksum "toolchain_key" }}
-#          paths:
-#            - ~/cache/<< parameters.toolchain >>
-      - run:
-          name: Setup build environment
-          command: |
-            echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV
-            # Install Ninja
-            NINJA_URL=https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip
-            wget $NINJA_URL -O ninja-linux.zip
-            unzip ninja-linux.zip -d ~/bin
-
-  get-deps:
-    parameters:
-      family:
-        type: string
-    steps:
-      - run:
-          name: Get Dependencies
-          command: |
-            python tools/get_deps.py << parameters.family >>
 
 jobs:
-  arm-clang:
+  set-matrix:
-    parameters:
+    executor: continuation/default
-      family:
-        type: string
-      build-system:
-        type: string
-
     docker:
       - image: cimg/base:current
-    resource_class: medium
+    resource_class: small
-    environment:
-      TOOLCHAIN_URL: https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz
     steps:
       - checkout
-      - setup-toolchain:
-          toolchain: clang
-          toolchain_url: $TOOLCHAIN_URL
-      - get-deps:
-          family: << parameters.family >>
       - run:
-          name: Build
+          name: Set matrix
           command: |
-            # Only build one board per family for non PRs i.e commit to master
+            MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py)
-            ONE_PER_FAMILY=""
+            echo "MATRIX_JSON=$MATRIX_JSON"
-            if [ -z "$CIRCLE_PULL_REQUEST" ]; then
+
-              ONE_PER_FAMILY="--one-per-family"
+            TOOLCHAIN_LIST=("arm-clang" "arm-gcc")
-            fi
+            for toolchain in "${TOOLCHAIN_LIST[@]}"; do
-            python tools/build.py $ONE_PER_FAMILY -s << parameters.build-system >> --toolchain clang << parameters.family >>
+              FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\".family")
+              echo "${toolchain}_FAMILY=$FAMILY"
+              echo "      - build:" >> .circleci/config2.yml
+              echo "          matrix:" >> .circleci/config2.yml
+              echo "            parameters:" >> .circleci/config2.yml
+              echo "              toolchain: ['$toolchain']" >> .circleci/config2.yml
+              echo "              build-system: ['cmake']" >> .circleci/config2.yml
+              echo "              family: $FAMILY" >> .circleci/config2.yml
+            done
+
+      - continuation/continue:
+          configuration_path: .circleci/config2.yml
 
 workflows:
-  build:
+  set-matrix:
     jobs:
-      - arm-clang:
+      - set-matrix
-          matrix:
-            parameters:
-              build-system:
-                - cmake
-              #family: ['stm32f1']
-              #family: ['stm32f1', 'stm32f2']
-              family: ['imxrt', 'kinetis_k kinetis_kl kinetis_k32l2', 'lpc11 lpc13 lpc15', 'lpc17 lpc18 lpc40 lpc43', 'lpc51 lpc54 lpc55', 'nrf', 'samd11 samd21 saml2x', 'samd5x_e5x samg', 'stm32f0 stm32f1 stm32f2 stm32f3', 'stm32f4', 'stm32f7', 'stm32g0 stm32g4 stm32h5', 'stm32h7', 'stm32l4 stm32u5 stm32wb']

.circleci/config2.yml

@@ -0,0 +1,98 @@
+version: 2.1
+
+commands:
+  setup-toolchain:
+    parameters:
+        toolchain:
+            type: string
+    steps:
+      - run:
+          name: Install Toolchain
+          command: |
+            TOOLCHAIN_JSON='{
+              "arm-clang": "https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz",
+              "arm-gcc": "https://github.com/xpack-dev-tools/arm-none-eabi-gcc-xpack/releases/download/v12.3.1-1.1/xpack-arm-none-eabi-gcc-12.3.1-1.1-linux-x64.tar.gz"
+            }'
+            toolchain_url=$(echo $TOOLCHAIN_JSON | jq -r '.["<< parameters.toolchain >>"]')
+            echo "toolchain_url=$toolchain_url"
+
+            # download and extract toolchain
+            mkdir -p ~/cache/<< parameters.toolchain >>
+            wget $toolchain_url -O toolchain.tar.gz
+            tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz
+
+            # Add toolchain to PATH
+            echo "export PATH=$PATH:`echo ~/cache/<< parameters.toolchain >>/*/bin`" >> $BASH_ENV
+
+  get-deps:
+    parameters:
+      family:
+        type: string
+    steps:
+      - run:
+          name: Get Dependencies
+          command: |
+            python tools/get_deps.py << parameters.family >>
+
+            # Install Pico SDK
+            if [ << parameters.family >> == "rp2040" ]; then
+              git clone --depth 1 https://github.com/raspberrypi/pico-sdk.git ~/pico-sdk
+              echo "export PICO_SDK_PATH=~/pico-sdk" >> $BASH_ENV
+            fi
+
+jobs:
+  build:
+    parameters:
+      build-system:
+        type: string
+      toolchain:
+        type: string
+      family:
+        type: string
+
+    docker:
+      - image: cimg/base:current
+    resource_class: medium+
+    steps:
+      - checkout
+      - when:
+          condition: << parameters.build-system >> == 'cmake'
+          steps:
+            - run:
+                name: Install Ninja
+                command: |
+                  # Install Ninja
+                  NINJA_URL=https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux.zip
+                  wget $NINJA_URL -O ninja-linux.zip
+                  unzip ninja-linux.zip -d ~/bin
+      - setup-toolchain:
+          toolchain: << parameters.toolchain >>
+      - get-deps:
+          family: << parameters.family >>
+      - run:
+          name: Build
+          command: |
+            # Only build one board per family for non PRs i.e commit to master
+            ONE_PER_FAMILY=""
+            if [ -z "$CIRCLE_PULL_REQUEST" ]; then
+              ONE_PER_FAMILY="--one-per-family"
+            fi
+
+            # Toolchain option default is gcc
+            if [ "<< parameters.toolchain >>" == "arm-clang" ]; then
+              TOOLCHAIN_OPTION="--toolchain clang"
+            elif [ "<< parameters.toolchain >>" == "arm-gcc" ]; then
+              TOOLCHAIN_OPTION="--toolchain gcc"
+            fi
+
+            python tools/build.py $ONE_PER_FAMILY -s << parameters.build-system >> $TOOLCHAIN_OPTION << parameters.family >>
+
+workflows:
+  build:
+    jobs:
+#      - build:
+#          matrix:
+#            parameters:
+#              toolchain: ['arm-clang']
+#              build-system: ['cmake']
+#              family: ['imxrt']

.github/workflows/build.yml

@@ -58,9 +58,8 @@ jobs:
       fail-fast: false
       matrix:
         toolchain:
-          # - 'arm-clang' is built by circle-ci
+          # - 'arm-clang' is built by circle-ci in PR
           - 'aarch64-gcc'
-          - 'arm-gcc'
           - 'msp430-gcc'
           - 'riscv-gcc'
     with:
@@ -69,6 +68,20 @@ jobs:
       build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }}
       one-per-family: ${{ github.event_name != 'pull_request' }}
 
+  # ---------------------------------------
+  # Build CMake arm-gcc
+  # only build with push, for PR: all board is built by circle-ci
+  # ---------------------------------------
+  cmake-arm-gcc:
+    if: github.event_name == 'push'
+    needs: set-matrix
+    uses: ./.github/workflows/build_util.yml
+    with:
+      build-system: 'cmake'
+      toolchain: 'arm-gcc'
+      build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)['arm-gcc'].family) }}
+      one-per-family: true
+
   # ---------------------------------------
   # Build Make
   # ---------------------------------------
@@ -80,7 +93,7 @@ jobs:
       fail-fast: false
       matrix:
         toolchain:
-          # 'arm-clang' is built by circle-ci
+          # 'arm-clang' would be built by circle-ci
           - 'aarch64-gcc'
           - 'arm-gcc'
           - 'msp430-gcc'

tools/build_utils.py

@@ -81,43 +81,6 @@ def skip_example(example, board):
     return False
 
 
-def build_example(example, board, make_option):
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
-
-    # succeeded, failed, skipped
-    ret = [0, 0, 0]
-
-    make_cmd = "make -j -C examples/{} BOARD={} {}".format(example, board, make_option)
-
-    # Check if board is skipped
-    if skip_example(example, board):
-        status = SKIPPED
-        ret[2] = 1
-        print(build_format.format(example, board, status, '-', flash_size, sram_size))
-    else:
-        #subprocess.run(make_cmd + " clean", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run(make_cmd + " all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if build_result.returncode == 0:
-            status = SUCCEEDED
-            ret[0] = 1
-            (flash_size, sram_size) = build_size(make_cmd)
-            #subprocess.run(make_cmd + " copy-artifact", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        else:
-            status = FAILED
-            ret[1] = 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-    return ret
-
-
 def build_size(make_cmd):
     size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines()
     for i, l in enumerate(size_output):
@@ -129,3 +92,38 @@ def build_size(make_cmd):
             return (flash_size, sram_size)
 
     return (0, 0)
+
+
+def build_example(example, board, make_option):
+    start_time = time.monotonic()
+    flash_size = "-"
+    sram_size = "-"
+
+    # succeeded, failed, skipped
+    ret = [0, 0, 0]
+
+    make_cmd = f"make -j -C examples/{example} BOARD={board} {make_option}"
+
+    # Check if board is skipped
+    if skip_example(example, board):
+        status = SKIPPED
+        ret[2] = 1
+        print(build_format.format(example, board, status, '-', flash_size, sram_size))
+    else:
+        build_result = subprocess.run(f"{make_cmd} all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        if build_result.returncode == 0:
+            status = SUCCEEDED
+            ret[0] = 1
+            (flash_size, sram_size) = build_size(make_cmd)
+        else:
+            status = FAILED
+            ret[1] = 1
+
+        build_duration = time.monotonic() - start_time
+        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
+
+        if build_result.returncode != 0:
+            print(build_result.stdout.decode("utf-8"))
+
+    return ret