使用多进程加快编译速度

make_riscv.py

@@ -3,6 +3,8 @@ import os
 import sys
 import time
 import shutil
+import dataclasses
+from multiprocessing import  Process,Queue
 
 
 
@@ -127,24 +129,70 @@ def read_depend_files(name:str):
     return t
 
 
+@dataclasses.dataclass
+class cmd_item_t:
+  cmd:str
+  info:str
+
+def run_cmd(cmd_queue:Queue,cpu_index:int,return_list:Queue):
+  while(not cmd_queue.empty()):
+    try:
+      cmd=cmd_queue.get_nowait()
+    except Exception:
+      return_list.put((cpu_index,True))
+      return
+    print(f"[{cpu_index}] {cmd.info}")
+    ret=os.system(cmd.cmd)
+    if(ret):
+      return_list.put((cpu_index,False))
+      return
+  return_list.put((cpu_index,True))
+
+
+def run_cmd_queue(cmd_queue:Queue,cpu_num:int=4):
+  if(cmd_queue.empty()):
+    return
+  process_list = []
+  return_list=Queue()
+  for i in range(cpu_num):
+    p = Process(target=run_cmd,args=(cmd_queue,i,return_list,))
+    p.start()
+    process_list.append(p)
+  for i in process_list:
+    p.join()
+  while not return_list.empty():
+    i=return_list.get()
+    if(not i[1]):
+      print(f"子进程 [{i[0]}] 运行失败")
+      exit(-1)
+
+
+# 保证目标都存在
+def check_exists(src:list):
+  for item in src:
+    for i in range(10):
+      if(os.path.exists(item)):
+        break
+      time.sleep(0.1)
 # 生成依赖关系
 def build_depend(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
   for i in src:
     name=os.path.splitext(tran_path(i))[0]
     dst='.'.join([name,'d'])
     if(check_rebuild(dst,[i])):
       cmd=f"{CC} -MM {i} -o {dst} {flags}"
-      print(f"更新 {dst}")
-      ret=os.system(cmd)
-      if(ret):
-        exit()
-    else:
-      # print(f"{i} 没有更新源文件")
-      pass
+      CmdQueue.put(cmd_item_t(cmd,f"更新 {dst}"))
+      dst_list.append(dst)
+  run_cmd_queue(CmdQueue)
+  check_exists(dst_list)
 
 # 生成中间文件
 def build_object(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
   for i in src:
     name_t=os.path.splitext(tran_path(i))
@@ -156,20 +204,14 @@ def build_object(src:list):
     if(file_type in ['.c','.C']):
       if(check_rebuild(dst,read_depend_files(cd))):
         cmd=f"{CC} -c {i} -o {dst} {flags}"
-      else:
-        # print(f"{i} 没有更新依赖关系")
-        pass
     elif(file_type in ['.s','.S','.asm','.ASM']):
       if(check_rebuild(dst,read_depend_files(cd))):
         cmd=f"{CC} -c {i} -o {dst} {flags}"
-      else:
-        # print(f"{i} 没有更新依赖关系")
-        pass
     if(len(cmd)>0):
-      print(f"编译 {dst}")
-      ret=os.system(cmd)
-      if(ret):
-        exit()
+      CmdQueue.put(cmd_item_t(cmd,f"编译 {dst}"))
+      dst_list.append(dst)
+  run_cmd_queue(CmdQueue)
+  check_exists(dst_list)
 
 
 # 生成可执行文件
@@ -186,9 +228,6 @@ def build_target(src:list):
     ret=os.system(cmd)
     if(ret):
       exit()
-  else:
-    # print(f"{dst} 没有更新的链接文件")
-    pass
 
 
 
@@ -208,4 +247,7 @@ def main():
 
 
 if __name__ == "__main__":
+  tick_start=time.time()
   main()
+  tick_end=time.time()
+  print(f"cost: {tick_end-tick_start}")