Rework ci (#2631)
* add name field to usbd_class_driver_t * ci: use set matrix py script * add riscv32 and cmake support for ch32v307, fomu, gd32vf103 * update build_cmake.py to take --family --board --toolchain * separate hil test to its own workflow * move esp32 board into separated hil json * add make build to ci * remov build_make.py * build.py support esp32 board * setup toolchain support esp-idf * fix missing click * merge family in matrix build to reduce jobs * skip cifuzz since it still has issue with get_deps and click
This commit is contained in:
Ha Thach
177
tools/build.py
Normal file
177
tools/build.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
import click
|
||||
from pathlib import Path
|
||||
from multiprocessing import Pool
|
||||
|
||||
import build_utils
|
||||
|
||||
SUCCEEDED = "\033[32msucceeded\033[0m"
|
||||
FAILED = "\033[31mfailed\033[0m"
|
||||
|
||||
build_separator = '-' * 106
|
||||
|
||||
|
||||
def run_cmd(cmd):
|
||||
#print(cmd)
|
||||
r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
title = 'command error'
|
||||
if r.returncode != 0:
|
||||
# print build output if failed
|
||||
if os.getenv('CI'):
|
||||
print(f"::group::{title}")
|
||||
print(r.stdout.decode("utf-8"))
|
||||
print(f"::endgroup::")
|
||||
else:
|
||||
print(title)
|
||||
print(r.stdout.decode("utf-8"))
|
||||
return r
|
||||
|
||||
def find_family(board):
|
||||
bsp_dir = Path("hw/bsp")
|
||||
for family_dir in bsp_dir.iterdir():
|
||||
if family_dir.is_dir():
|
||||
board_dir = family_dir / 'boards' / board
|
||||
if board_dir.exists():
|
||||
return family_dir.name
|
||||
return None
|
||||
|
||||
|
||||
def get_examples(family):
|
||||
all_examples = []
|
||||
for d in os.scandir("examples"):
|
||||
if d.is_dir() and 'cmake' not in d.name and 'build_system' not in d.name:
|
||||
for entry in os.scandir(d.path):
|
||||
if entry.is_dir() and 'cmake' not in entry.name:
|
||||
if family != 'espressif' or 'freertos' in entry.name:
|
||||
all_examples.append(d.name + '/' + entry.name)
|
||||
|
||||
if family == 'espressif':
|
||||
all_examples.append('device/board_test')
|
||||
all_examples.append('device/video_capture')
|
||||
all_examples.sort()
|
||||
return all_examples
|
||||
|
||||
|
||||
def build_board_cmake(board, toolchain):
|
||||
start_time = time.monotonic()
|
||||
ret = [0, 0, 0]
|
||||
|
||||
build_dir = f"cmake-build/cmake-build-{board}"
|
||||
family = find_family(board)
|
||||
if family == 'espressif':
|
||||
# for espressif, we have to build example individually
|
||||
all_examples = get_examples(family)
|
||||
for example in all_examples:
|
||||
r = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" -DBOARD={board} -DMAX3421_HOST=1')
|
||||
if r.returncode == 0:
|
||||
r = run_cmd(f'cmake --build {build_dir}/{example}')
|
||||
if r.returncode == 0:
|
||||
ret[0] += 1
|
||||
else:
|
||||
ret[1] += 1
|
||||
else:
|
||||
r = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel -DTOOLCHAIN={toolchain}')
|
||||
if r.returncode == 0:
|
||||
r = run_cmd(f"cmake --build {build_dir}")
|
||||
if r.returncode == 0:
|
||||
ret[0] += 1
|
||||
else:
|
||||
ret[1] += 1
|
||||
|
||||
duration = time.monotonic() - start_time
|
||||
|
||||
if ret[1] == 0:
|
||||
status = SUCCEEDED
|
||||
else:
|
||||
status = FAILED
|
||||
|
||||
flash_size = "-"
|
||||
sram_size = "-"
|
||||
example = 'all'
|
||||
title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
|
||||
print(title)
|
||||
return ret
|
||||
|
||||
|
||||
def build_family(family, toolchain, build_system):
|
||||
all_boards = []
|
||||
for entry in os.scandir(f"hw/bsp/{family}/boards"):
|
||||
if entry.is_dir() and entry.name != 'pico_sdk':
|
||||
all_boards.append(entry.name)
|
||||
all_boards.sort()
|
||||
|
||||
# success, failed, skipped
|
||||
ret = [0, 0, 0]
|
||||
if build_system == 'cmake':
|
||||
for board in all_boards:
|
||||
if build_board_cmake(board, toolchain):
|
||||
ret[0] += 1
|
||||
else:
|
||||
ret[1] += 1
|
||||
elif build_system == 'make':
|
||||
all_examples = get_examples(family)
|
||||
for example in all_examples:
|
||||
with Pool(processes=os.cpu_count()) as pool:
|
||||
pool_args = list((map(lambda b, e=example, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_boards)))
|
||||
r = pool.starmap(build_utils.build_example, pool_args)
|
||||
# sum all element of same index (column sum)
|
||||
rsum = list(map(sum, list(zip(*r))))
|
||||
ret[0] += rsum[0]
|
||||
ret[1] += rsum[1]
|
||||
ret[2] += rsum[2]
|
||||
return ret
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('families', nargs=-1, required=False)
|
||||
@click.option('-b', '--board', multiple=True, default=None, help='Boards to build')
|
||||
@click.option('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
|
||||
@click.option('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
|
||||
def main(families, board, toolchain, build_system):
|
||||
if len(families) == 0 and len(board) == 0:
|
||||
print("Please specify families or board to build")
|
||||
return 1
|
||||
|
||||
print(build_separator)
|
||||
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
|
||||
total_time = time.monotonic()
|
||||
total_result = [0, 0, 0]
|
||||
|
||||
# build families: cmake, make
|
||||
if families is not None:
|
||||
all_families = []
|
||||
if 'all' in families:
|
||||
for entry in os.scandir("hw/bsp"):
|
||||
if entry.is_dir() and entry.name != 'espressif' and os.path.isfile(entry.path + "/family.cmake"):
|
||||
all_families.append(entry.name)
|
||||
else:
|
||||
all_families = list(families)
|
||||
all_families.sort()
|
||||
|
||||
# succeeded, failed
|
||||
for f in all_families:
|
||||
fret = build_family(f, toolchain, build_system)
|
||||
total_result[0] += fret[0]
|
||||
total_result[1] += fret[1]
|
||||
total_result[2] += fret[2]
|
||||
|
||||
# build board (only cmake)
|
||||
if board is not None:
|
||||
for b in board:
|
||||
r = build_board_cmake(b, toolchain)
|
||||
total_result[0] += r[0]
|
||||
total_result[1] += r[1]
|
||||
total_result[2] += r[2]
|
||||
|
||||
total_time = time.monotonic() - total_time
|
||||
print(build_separator)
|
||||
print(f"Build Summary: {total_result[0]} {SUCCEEDED}, {total_result[1]} {FAILED} and took {total_time:.2f}s")
|
||||
print(build_separator)
|
||||
return total_result[1]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,69 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
from multiprocessing import Pool
|
||||
|
||||
import build_utils
|
||||
|
||||
SUCCEEDED = "\033[32msucceeded\033[0m"
|
||||
FAILED = "\033[31mfailed\033[0m"
|
||||
SKIPPED = "\033[33mskipped\033[0m"
|
||||
|
||||
build_separator = '-' * 106
|
||||
|
||||
|
||||
def filter_with_input(mylist):
|
||||
if len(sys.argv) > 1:
|
||||
input_args = list(set(mylist).intersection(sys.argv))
|
||||
if len(input_args) > 0:
|
||||
mylist[:] = input_args
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If examples are not specified in arguments, build all
|
||||
all_examples = []
|
||||
for dir1 in os.scandir("examples"):
|
||||
if dir1.is_dir():
|
||||
for entry in os.scandir(dir1.path):
|
||||
if entry.is_dir():
|
||||
all_examples.append(dir1.name + '/' + entry.name)
|
||||
filter_with_input(all_examples)
|
||||
all_examples.sort()
|
||||
|
||||
# If boards are not specified in arguments, build all
|
||||
all_boards = []
|
||||
for entry in os.scandir("hw/bsp"):
|
||||
if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
|
||||
all_boards.append(entry.name)
|
||||
filter_with_input(all_boards)
|
||||
all_boards.sort()
|
||||
|
||||
# Get dependencies
|
||||
for b in all_boards:
|
||||
subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(b), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
print(build_separator)
|
||||
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
|
||||
total_time = time.monotonic()
|
||||
|
||||
# succeeded, failed, skipped
|
||||
total_result = [0, 0, 0]
|
||||
for example in all_examples:
|
||||
print(build_separator)
|
||||
with Pool(processes=os.cpu_count()) as pool:
|
||||
pool_args = list((map(lambda b, e=example, o='': [e, b, o], all_boards)))
|
||||
result = pool.starmap(build_utils.build_example, pool_args)
|
||||
# sum all element of same index (column sum)
|
||||
result = list(map(sum, list(zip(*result))))
|
||||
|
||||
# add to total result
|
||||
total_result = list(map(lambda x, y: x + y, total_result, result))
|
||||
|
||||
total_time = time.monotonic() - total_time
|
||||
print(build_separator)
|
||||
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
|
||||
FAILED, total_result[2], SKIPPED, total_time))
|
||||
print(build_separator)
|
||||
|
||||
sys.exit(total_result[1])
|
||||
@@ -1,105 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
import pathlib
|
||||
from multiprocessing import Pool
|
||||
|
||||
import build_utils
|
||||
|
||||
SUCCEEDED = "\033[32msucceeded\033[0m"
|
||||
FAILED = "\033[31mfailed\033[0m"
|
||||
SKIPPED = "\033[33mskipped\033[0m"
|
||||
|
||||
build_separator = '-' * 106
|
||||
|
||||
def filter_with_input(mylist):
|
||||
if len(sys.argv) > 1:
|
||||
input_args = list(set(mylist).intersection(sys.argv))
|
||||
if len(input_args) > 0:
|
||||
mylist[:] = input_args
|
||||
|
||||
|
||||
def build_family(family, cmake_option):
|
||||
all_boards = []
|
||||
for entry in os.scandir("hw/bsp/{}/boards".format(family)):
|
||||
if entry.is_dir() and entry.name != 'pico_sdk':
|
||||
all_boards.append(entry.name)
|
||||
all_boards.sort()
|
||||
|
||||
# success, failed, skipped
|
||||
ret = [0, 0, 0]
|
||||
for board in all_boards:
|
||||
start_time = time.monotonic()
|
||||
|
||||
build_dir = f"cmake-build/cmake-build-{board}"
|
||||
|
||||
# Generate build
|
||||
r = subprocess.run(f"cmake examples -B {build_dir} -G \"Ninja\" -DFAMILY={family} -DBOARD"
|
||||
f"={board} -DCMAKE_BUILD_TYPE=MinSizeRel {cmake_option}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
|
||||
# Build
|
||||
if r.returncode == 0:
|
||||
r = subprocess.run(f"cmake --build {build_dir}", shell=True, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
duration = time.monotonic() - start_time
|
||||
|
||||
if r.returncode == 0:
|
||||
status = SUCCEEDED
|
||||
ret[0] += 1
|
||||
else:
|
||||
status = FAILED
|
||||
ret[1] += 1
|
||||
|
||||
flash_size = "-"
|
||||
sram_size = "-"
|
||||
example = 'all'
|
||||
title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
|
||||
|
||||
if os.getenv('CI'):
|
||||
# always print build output if in CI
|
||||
print(f"::group::{title}")
|
||||
print(r.stdout.decode("utf-8"))
|
||||
print(f"::endgroup::")
|
||||
else:
|
||||
# print build output if failed
|
||||
print(title)
|
||||
if r.returncode != 0:
|
||||
print(r.stdout.decode("utf-8"))
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
cmake_options = ''
|
||||
for a in sys.argv[1:]:
|
||||
if a.startswith('-'):
|
||||
cmake_options += ' ' + a
|
||||
|
||||
# If family are not specified in arguments, build all supported
|
||||
all_families = []
|
||||
for entry in os.scandir("hw/bsp"):
|
||||
if entry.is_dir() and entry.name != 'espressif' and os.path.isfile(entry.path + "/family.cmake"):
|
||||
all_families.append(entry.name)
|
||||
filter_with_input(all_families)
|
||||
all_families.sort()
|
||||
|
||||
print(build_separator)
|
||||
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
|
||||
total_time = time.monotonic()
|
||||
|
||||
# succeeded, failed, skipped
|
||||
total_result = [0, 0, 0]
|
||||
for family in all_families:
|
||||
fret = build_family(family, cmake_options)
|
||||
if len(fret) == len(total_result):
|
||||
total_result = [total_result[i] + fret[i] for i in range(len(fret))]
|
||||
|
||||
total_time = time.monotonic() - total_time
|
||||
print(build_separator)
|
||||
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
|
||||
FAILED, total_result[2], SKIPPED, total_time))
|
||||
print(build_separator)
|
||||
|
||||
sys.exit(total_result[1])
|
||||
@@ -1,80 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from multiprocessing import Pool
|
||||
|
||||
import build_utils
|
||||
|
||||
SUCCEEDED = "\033[32msucceeded\033[0m"
|
||||
FAILED = "\033[31mfailed\033[0m"
|
||||
SKIPPED = "\033[33mskipped\033[0m"
|
||||
|
||||
build_separator = '-' * 106
|
||||
|
||||
|
||||
def filter_with_input(mylist):
|
||||
if len(sys.argv) > 1:
|
||||
input_args = list(set(mylist).intersection(sys.argv))
|
||||
if len(input_args) > 0:
|
||||
mylist[:] = input_args
|
||||
|
||||
|
||||
def build_family(example, family, make_option):
|
||||
all_boards = []
|
||||
for entry in os.scandir("hw/bsp/{}/boards".format(family)):
|
||||
if entry.is_dir() and entry.name != 'pico_sdk':
|
||||
all_boards.append(entry.name)
|
||||
filter_with_input(all_boards)
|
||||
all_boards.sort()
|
||||
|
||||
with Pool(processes=os.cpu_count()) as pool:
|
||||
pool_args = list((map(lambda b, e=example, o=make_option: [e, b, o], all_boards)))
|
||||
result = pool.starmap(build_utils.build_example, pool_args)
|
||||
# sum all element of same index (column sum)
|
||||
return list(map(sum, list(zip(*result))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
make_option = ''
|
||||
for a in sys.argv:
|
||||
if 'TOOLCHAIN=' in sys.argv:
|
||||
make_option += ' ' + a
|
||||
|
||||
# If examples are not specified in arguments, build all
|
||||
all_examples = []
|
||||
for d in os.scandir("examples"):
|
||||
if d.is_dir() and 'cmake' not in d.name and 'build_system' not in d.name:
|
||||
for entry in os.scandir(d.path):
|
||||
if entry.is_dir() and 'cmake' not in entry.name:
|
||||
all_examples.append(d.name + '/' + entry.name)
|
||||
filter_with_input(all_examples)
|
||||
all_examples.sort()
|
||||
|
||||
# If family are not specified in arguments, build all
|
||||
all_families = []
|
||||
for entry in os.scandir("hw/bsp"):
|
||||
if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name != 'espressif':
|
||||
all_families.append(entry.name)
|
||||
filter_with_input(all_families)
|
||||
all_families.sort()
|
||||
|
||||
print(build_separator)
|
||||
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
|
||||
total_time = time.monotonic()
|
||||
|
||||
# succeeded, failed, skipped
|
||||
total_result = [0, 0, 0]
|
||||
for example in all_examples:
|
||||
print(build_separator)
|
||||
for family in all_families:
|
||||
fret = build_family(example, family, make_option)
|
||||
if len(fret) == len(total_result):
|
||||
total_result = [total_result[i] + fret[i] for i in range(len(fret))]
|
||||
|
||||
total_time = time.monotonic() - total_time
|
||||
print(build_separator)
|
||||
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
|
||||
FAILED, total_result[2], SKIPPED, total_time))
|
||||
print(build_separator)
|
||||
|
||||
sys.exit(total_result[1])
|
||||
@@ -1,3 +1,4 @@
|
||||
import click
|
||||
import sys
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
@@ -227,27 +228,46 @@ def get_a_dep(d):
|
||||
return 0
|
||||
|
||||
|
||||
# Arguments can be
|
||||
# - family name
|
||||
# - specific deps path
|
||||
# - all
|
||||
if __name__ == "__main__":
|
||||
def find_family(board):
|
||||
bsp_dir = Path(TOP / "hw/bsp")
|
||||
for family_dir in bsp_dir.iterdir():
|
||||
if family_dir.is_dir():
|
||||
board_dir = family_dir / 'boards' / board
|
||||
if board_dir.exists():
|
||||
return family_dir.name
|
||||
return None
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('family', nargs=-1, required=False)
|
||||
@click.option('-b', '--board', multiple=True, default=None, help='Boards to fetch')
|
||||
def main(family, board):
|
||||
if len(family) == 0 and len(board) == 0:
|
||||
print("Please specify family or board to fetch")
|
||||
return
|
||||
|
||||
status = 0
|
||||
deps = list(deps_mandatory.keys())
|
||||
# get all if 'all' is argument
|
||||
if len(sys.argv) == 2 and sys.argv[1] == 'all':
|
||||
|
||||
if 'all' in family:
|
||||
deps += deps_optional.keys()
|
||||
else:
|
||||
for arg in sys.argv[1:]:
|
||||
if arg in deps_all.keys():
|
||||
# if arg is a dep, add it
|
||||
deps.append(arg)
|
||||
else:
|
||||
# arg is a family name, add all deps of that family
|
||||
for d in deps_optional:
|
||||
if arg in deps_optional[d][2]:
|
||||
deps.append(d)
|
||||
family = list(family)
|
||||
if board is not None:
|
||||
for b in board:
|
||||
f = find_family(b)
|
||||
if f is not None:
|
||||
family.append(f)
|
||||
|
||||
for f in family:
|
||||
for d in deps_optional:
|
||||
if f in deps_optional[d][2]:
|
||||
deps.append(d)
|
||||
|
||||
with Pool() as pool:
|
||||
status = sum(pool.map(get_a_dep, deps))
|
||||
sys.exit(status)
|
||||
return status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
Reference in New Issue
Block a user