update build_board.py to parallel build

tools/build_board.py

@@ -1,8 +1,7 @@
 import os
-import glob
 import sys
-import subprocess
 import time
+from multiprocessing import Pool
 
 import build_utils
 
@@ -10,22 +9,17 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
 FAILED = "\033[31mfailed\033[0m"
 SKIPPED = "\033[33mskipped\033[0m"
 
-success_count = 0
-fail_count = 0
-skip_count = 0
-exit_status = 0
-
-total_time = time.monotonic()
-
-build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
+
 def filter_with_input(mylist):
     if len(sys.argv) > 1:
         input_args = list(set(mylist).intersection(sys.argv))
         if len(input_args) > 0:
             mylist[:] = input_args
 
+
+if __name__ == '__main__':
     # If examples are not specified in arguments, build all
     all_examples = []
     for dir1 in os.scandir("examples"):
@@ -44,56 +38,27 @@ for entry in os.scandir("hw/bsp"):
     filter_with_input(all_boards)
     all_boards.sort()
 
-def build_board(example, board):
-    global success_count, fail_count, skip_count, exit_status
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
-
-    # Check if board is skipped
-    if build_utils.skip_example(example, board):
-        success = SKIPPED
-        skip_count += 1
-        print(build_format.format(example, board, success, '-', flash_size, sram_size))
-    else:
-        subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if build_result.returncode == 0:
-            success = SUCCEEDED
-            success_count += 1
-            (flash_size, sram_size) = build_size(example, board)
-        else:
-            exit_status = build_result.returncode
-            success = FAILED
-            fail_count += 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-def build_size(example, board):
-    #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
-    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
-    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
-    size_list = size_output.split('\n')[1].split('\t')
-    flash_size = int(size_list[0])
-    sram_size = int(size_list[1]) + int(size_list[2])
-    return (flash_size, sram_size)
-
     print(build_separator)
-print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    total_time = time.monotonic()
 
+    # succeeded, failed, skipped
+    total_result = [0, 0, 0]
     for example in all_examples:
         print(build_separator)
-    for board in all_boards:
+        with Pool(processes=os.cpu_count()) as pool:
-        build_board(example, board)
+            pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
+            result = pool.starmap(build_utils.build_example, pool_args)
+            # sum all element of same index (column sum)
+            result = list(map(sum, list(zip(*result))))
+            
+            # add to total result
+            total_result = list(map(lambda x, y: x + y, total_result, result))
 
     total_time = time.monotonic() - total_time
     print(build_separator)
-print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
+    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
+                                                                       FAILED, total_result[2], SKIPPED, total_time))
     print(build_separator)
 
-sys.exit(exit_status)
+    sys.exit(total_result[1])

tools/build_family.py

@@ -1,7 +1,5 @@
 import os
-import glob
 import sys
-import subprocess
 import time
 from multiprocessing import Pool
 
@@ -11,15 +9,32 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
 FAILED = "\033[31mfailed\033[0m"
 SKIPPED = "\033[33mskipped\033[0m"
 
-build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
+
 def filter_with_input(mylist):
     if len(sys.argv) > 1:
         input_args = list(set(mylist).intersection(sys.argv))
         if len(input_args) > 0:
             mylist[:] = input_args
 
+
+def build_family(example, family):
+    all_boards = []
+    for entry in os.scandir("hw/bsp/{}/boards".format(family)):
+        if entry.is_dir() and entry.name != 'pico_sdk':
+            all_boards.append(entry.name)
+    filter_with_input(all_boards)
+    all_boards.sort()
+
+    with Pool(processes=os.cpu_count()) as pool:
+        pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
+        result = pool.starmap(build_utils.build_example, pool_args)
+        # sum all element of same index (column sum)
+        return list(map(sum, list(zip(*result))))
+
+
+if __name__ == '__main__':
     # If examples are not specified in arguments, build all
     all_examples = []
     for dir1 in os.scandir("examples"):
@@ -35,73 +50,15 @@ all_families = []
     for entry in os.scandir("hw/bsp"):
         if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
             all_families.append(entry.name)
-            
     filter_with_input(all_families)
     all_families.sort()
 
-def build_family(example, family):
-    all_boards = []
-    for entry in os.scandir("hw/bsp/{}/boards".format(family)):
-        if entry.is_dir() and entry.name != 'pico_sdk':
-            all_boards.append(entry.name)
-    filter_with_input(all_boards)
-    all_boards.sort()
-
-    with Pool(processes=len(all_boards)) as pool:
-        pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
-        result = pool.starmap(build_board, pool_args)
-        return list(map(sum, list(zip(*result))))
-    
-def build_board(example, board):
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
-
-    # succeeded, failed, skipped
-    ret = [0, 0, 0]
-
-    # Check if board is skipped
-    if build_utils.skip_example(example, board):
-        status = SKIPPED
-        ret[2] = 1
-        print(build_format.format(example, board, status, '-', flash_size, sram_size))
-    else:   
-        #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if build_result.returncode == 0:
-            status = SUCCEEDED
-            ret[0] = 1
-            (flash_size, sram_size) = build_size(example, board)
-            subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        else:
-            status = FAILED
-            ret[1] = 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-    return ret
-
-def build_size(example, board):
-    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
-    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
-    size_list = size_output.split('\n')[1].split('\t')
-    flash_size = int(size_list[0])
-    sram_size = int(size_list[1]) + int(size_list[2])
-    return (flash_size, sram_size)
-
-if __name__ == '__main__':
-    # succeeded, failed, skipped
-    total_result = [0, 0, 0]
-
     print(build_separator)
-    print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
     total_time = time.monotonic()
 
+    # succeeded, failed, skipped
+    total_result = [0, 0, 0]
     for example in all_examples:
         print(build_separator)
         for family in all_families:
@@ -110,7 +67,8 @@ if __name__ == '__main__':
 
     total_time = time.monotonic() - total_time
     print(build_separator)
-    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time))
+    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
+                                                                       FAILED, total_result[2], SKIPPED, total_time))
     print(build_separator)
 
     sys.exit(total_result[1])

tools/build_utils.py

@@ -1,4 +1,13 @@
+import subprocess
 import pathlib
+import time
+
+build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
+
+SUCCEEDED = "\033[32msucceeded\033[0m"
+FAILED = "\033[31mfailed\033[0m"
+SKIPPED = "\033[33mskipped\033[0m"
+
 
 def skip_example(example, board):
     ex_dir = pathlib.Path('examples/') / example
@@ -66,3 +75,48 @@ def skip_example(example, board):
                     "family:" + family in onlys)
 
     return False
+
+
+def build_example(example, board):
+    start_time = time.monotonic()
+    flash_size = "-"
+    sram_size = "-"
+
+    # succeeded, failed, skipped
+    ret = [0, 0, 0]
+
+    # Check if board is skipped
+    if skip_example(example, board):
+        status = SKIPPED
+        ret[2] = 1
+        print(build_format.format(example, board, status, '-', flash_size, sram_size))
+    else:
+        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True,
+                                      stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        if build_result.returncode == 0:
+            status = SUCCEEDED
+            ret[0] = 1
+            (flash_size, sram_size) = build_size(example, board)
+            subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True,
+                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        else:
+            status = FAILED
+            ret[1] = 1
+
+        build_duration = time.monotonic() - start_time
+        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
+
+        if build_result.returncode != 0:
+            print(build_result.stdout.decode("utf-8"))
+
+    return ret
+
+
+def build_size(example, board):
+    elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
+    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
+    size_list = size_output.split('\n')[1].split('\t')
+    flash_size = int(size_list[0])
+    sram_size = int(size_list[1]) + int(size_list[2])
+    return (flash_size, sram_size)