使用多进程加快编译速度

make.py

@@ -6,6 +6,8 @@
 import os
 import sys
 import time
+import dataclasses
+from multiprocessing import  Process,Queue
 from create_lambda_fun import search_lambda
 from create_signal_fun import moc_file_create
 
@@ -18,12 +20,6 @@ from create_signal_fun import moc_file_create
 
 
 CC = 'gcc'
-# CC = 'C:\\ARM_GCC\\bin\\arm-none-eabi-gcc'
-
-# AS = CC + ' -x assembler-with-cpp'
-
-# HEX = 'C:\\ARM_GCC\\bin\\arm-none-eabi-objcopy' + ' -O ihex'
-# BIN = 'C:\\ARM_GCC\\bin\\arm-none-eabi-objcopy' + ' -O binary -S'
 
 CSRC = ["main.c"]
 
@@ -84,7 +80,6 @@ def tran_path(path:str):
 
 # 判断是否需要重新生成
 def check_rebuild(dst:str,src:list):
-  # print(f"src:{src}")
   if(not os.path.exists(dst)):
     return True
   dst_time=os.path.getmtime(dst)
@@ -94,9 +89,6 @@ def check_rebuild(dst:str,src:list):
   src_time.sort()
   if(src_time[-1]>dst_time):
     return True
-  # for item in src_time:
-  #   if(item>dst_time):
-  #     return True
   return False
 
 
@@ -114,28 +106,76 @@ def read_depend_files(name:str):
         t+=line
     t=t.split(':')[-1].strip()
     t=t.split(' ')
-    # print(f"依赖列表{t}")
     return t
 
 
+@dataclasses.dataclass
+class cmd_item_t:
+  cmd:str
+  info:str
+
+def run_cmd(cmd_queue:Queue,cpu_index:int,return_list:Queue):
+  while(not cmd_queue.empty()):
+    try:
+      cmd=cmd_queue.get_nowait()
+    except Exception:
+      return_list.put((cpu_index,True))
+      return
+    print(f"[{cpu_index}] {cmd.info}")
+    ret=os.system(cmd.cmd)
+    if(ret):
+      return_list.put((cpu_index,False))
+      return
+  return_list.put((cpu_index,True))
+
+
+def run_cmd_queue(cmd_queue:Queue,cpu_num:int=4):
+  if(cmd_queue.empty()):
+    return
+  process_list = []
+  return_list=Queue()
+  for i in range(cpu_num):
+    p = Process(target=run_cmd,args=(cmd_queue,i,return_list,))
+    p.start()
+    process_list.append(p)
+  for i in process_list:
+    p.join()
+  while not return_list.empty():
+    i=return_list.get()
+    if(not i[1]):
+      print(f"子进程 [{i[0]}] 运行失败")
+      exit(-1)
+
+
+# 保证目标都存在
+def check_exists(src:list):
+  for item in src:
+    for i in range(10):
+      if(os.path.exists(item)):
+        break
+      time.sleep(0.1)
+
+
 # 生成依赖关系
 def build_depend(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(CINC)} {' '.join(CDEF)} {' '.join(CFLAG)}"
   for i in src:
     name=os.path.splitext(tran_path(i))[0]
     dst='.'.join([name,'d'])
     if(check_rebuild(dst,[i])):
       cmd=f"{CC} -MM {i} -o {dst} {flags}"
-      print(f"更新 {dst}")
+      CmdQueue.put(cmd_item_t(cmd,f"更新 {dst}"))
-      ret=os.system(cmd)
+      dst_list.append(dst)
-      if(ret):
+  run_cmd_queue(CmdQueue)
-        exit()
+  check_exists(dst_list)
-    else:
+
-      # print(f"{i} 没有更新源文件")
-      pass
 
 # 生成中间文件
 def build_object(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(CINC)} {' '.join(CDEF)} {' '.join(CFLAG)}"
   for i in src:
     name_t=os.path.splitext(tran_path(i))
@@ -143,25 +183,13 @@ def build_object(src:list):
     file_type=name_t[-1]
     dst='.'.join([name,'o'])
     cd='.'.join([name,'d'])
-    cmd = ''
     if(file_type in ['.c','.C']):
       if(check_rebuild(dst,read_depend_files(cd))):
         cmd=f"{CC} -c {i} -o {dst} {flags}"
-      else:
+        CmdQueue.put(cmd_item_t(cmd,f"编译 {dst}"))
-        # print(f"{i} 没有更新依赖关系")
+        dst_list.append(dst)
-        pass
+  run_cmd_queue(CmdQueue)
-    elif(file_type in ['.s','.S','.asm','.ASM']):
+  check_exists(dst_list)
-      if(check_rebuild(dst,[i])):
-        # cmd=f"{AS} -c {i} -o {dst} {flags}"
-        pass
-      else:
-        # print(f"{i} 没有更新依赖关系")
-        pass
-    if(len(cmd)>0):
-      print(f"编译 {dst}")
-      ret=os.system(cmd)
-      if(ret):
-        exit()
 
 
 # 生成可执行文件
@@ -178,9 +206,6 @@ def build_target(src:list):
     ret=os.system(cmd)
     if(ret):
       exit()
-  else:
-    # print(f"{dst} 没有更新的链接文件")
-    pass
 
 
 def main():
@@ -195,16 +220,17 @@ def main():
   CSRC=search_lambda(CSRC)
   moc_file_create(f"{BUILD_DIR}/moc_tmp.c",list(item[2:] for item in CINC))
   CSRC.append(f"{BUILD_DIR}/moc_tmp.c")
-  # ASRC+=find_type('./',['s','S','asm','ASM'])
 
 
   build_depend(CSRC)
   build_object(CSRC)
-  # build_object(ASRC)
   build_target(CSRC+ASRC)
 
 if __name__ == "__main__":
+  tick_start=time.time()
   main()
+  tick_end=time.time()
+  print(f"cost: {tick_end-tick_start}")
 
 
 

make_riscv.py

@@ -3,6 +3,8 @@ import os
 import sys
 import time
 import shutil
+import dataclasses
+from multiprocessing import  Process,Queue
 
 
 
@@ -127,24 +129,70 @@ def read_depend_files(name:str):
     return t
 
 
+@dataclasses.dataclass
+class cmd_item_t:
+  cmd:str
+  info:str
+
+def run_cmd(cmd_queue:Queue,cpu_index:int,return_list:Queue):
+  while(not cmd_queue.empty()):
+    try:
+      cmd=cmd_queue.get_nowait()
+    except Exception:
+      return_list.put((cpu_index,True))
+      return
+    print(f"[{cpu_index}] {cmd.info}")
+    ret=os.system(cmd.cmd)
+    if(ret):
+      return_list.put((cpu_index,False))
+      return
+  return_list.put((cpu_index,True))
+
+
+def run_cmd_queue(cmd_queue:Queue,cpu_num:int=4):
+  if(cmd_queue.empty()):
+    return
+  process_list = []
+  return_list=Queue()
+  for i in range(cpu_num):
+    p = Process(target=run_cmd,args=(cmd_queue,i,return_list,))
+    p.start()
+    process_list.append(p)
+  for i in process_list:
+    p.join()
+  while not return_list.empty():
+    i=return_list.get()
+    if(not i[1]):
+      print(f"子进程 [{i[0]}] 运行失败")
+      exit(-1)
+
+
+# 保证目标都存在
+def check_exists(src:list):
+  for item in src:
+    for i in range(10):
+      if(os.path.exists(item)):
+        break
+      time.sleep(0.1)
 # 生成依赖关系
 def build_depend(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
   for i in src:
     name=os.path.splitext(tran_path(i))[0]
     dst='.'.join([name,'d'])
     if(check_rebuild(dst,[i])):
       cmd=f"{CC} -MM {i} -o {dst} {flags}"
-      print(f"更新 {dst}")
+      CmdQueue.put(cmd_item_t(cmd,f"更新 {dst}"))
-      ret=os.system(cmd)
+      dst_list.append(dst)
-      if(ret):
+  run_cmd_queue(CmdQueue)
-        exit()
+  check_exists(dst_list)
-    else:
-      # print(f"{i} 没有更新源文件")
-      pass
 
 # 生成中间文件
 def build_object(src:list):
+  CmdQueue=Queue()
+  dst_list=[]
   flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
   for i in src:
     name_t=os.path.splitext(tran_path(i))
@@ -156,20 +204,14 @@ def build_object(src:list):
     if(file_type in ['.c','.C']):
       if(check_rebuild(dst,read_depend_files(cd))):
         cmd=f"{CC} -c {i} -o {dst} {flags}"
-      else:
-        # print(f"{i} 没有更新依赖关系")
-        pass
     elif(file_type in ['.s','.S','.asm','.ASM']):
       if(check_rebuild(dst,read_depend_files(cd))):
         cmd=f"{CC} -c {i} -o {dst} {flags}"
-      else:
-        # print(f"{i} 没有更新依赖关系")
-        pass
     if(len(cmd)>0):
-      print(f"编译 {dst}")
+      CmdQueue.put(cmd_item_t(cmd,f"编译 {dst}"))
-      ret=os.system(cmd)
+      dst_list.append(dst)
-      if(ret):
+  run_cmd_queue(CmdQueue)
-        exit()
+  check_exists(dst_list)
 
 
 # 生成可执行文件
@@ -186,9 +228,6 @@ def build_target(src:list):
     ret=os.system(cmd)
     if(ret):
       exit()
-  else:
-    # print(f"{dst} 没有更新的链接文件")
-    pass
 
 
 
@@ -208,4 +247,7 @@ def main():
 
 
 if __name__ == "__main__":
+  tick_start=time.time()
   main()
+  tick_end=time.time()
+  print(f"cost: {tick_end-tick_start}")