使用多进程加快编译速度
This commit is contained in:
106
make.py
106
make.py
@@ -6,6 +6,8 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import dataclasses
|
||||
from multiprocessing import Process,Queue
|
||||
from create_lambda_fun import search_lambda
|
||||
from create_signal_fun import moc_file_create
|
||||
|
||||
@@ -18,12 +20,6 @@ from create_signal_fun import moc_file_create
|
||||
|
||||
|
||||
CC = 'gcc'
|
||||
# CC = 'C:\\ARM_GCC\\bin\\arm-none-eabi-gcc'
|
||||
|
||||
# AS = CC + ' -x assembler-with-cpp'
|
||||
|
||||
# HEX = 'C:\\ARM_GCC\\bin\\arm-none-eabi-objcopy' + ' -O ihex'
|
||||
# BIN = 'C:\\ARM_GCC\\bin\\arm-none-eabi-objcopy' + ' -O binary -S'
|
||||
|
||||
CSRC = ["main.c"]
|
||||
|
||||
@@ -84,7 +80,6 @@ def tran_path(path:str):
|
||||
|
||||
# 判断是否需要重新生成
|
||||
def check_rebuild(dst:str,src:list):
|
||||
# print(f"src:{src}")
|
||||
if(not os.path.exists(dst)):
|
||||
return True
|
||||
dst_time=os.path.getmtime(dst)
|
||||
@@ -94,9 +89,6 @@ def check_rebuild(dst:str,src:list):
|
||||
src_time.sort()
|
||||
if(src_time[-1]>dst_time):
|
||||
return True
|
||||
# for item in src_time:
|
||||
# if(item>dst_time):
|
||||
# return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -114,28 +106,76 @@ def read_depend_files(name:str):
|
||||
t+=line
|
||||
t=t.split(':')[-1].strip()
|
||||
t=t.split(' ')
|
||||
# print(f"依赖列表{t}")
|
||||
return t
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class cmd_item_t:
|
||||
cmd:str
|
||||
info:str
|
||||
|
||||
def run_cmd(cmd_queue:Queue,cpu_index:int,return_list:Queue):
|
||||
while(not cmd_queue.empty()):
|
||||
try:
|
||||
cmd=cmd_queue.get_nowait()
|
||||
except Exception:
|
||||
return_list.put((cpu_index,True))
|
||||
return
|
||||
print(f"[{cpu_index}] {cmd.info}")
|
||||
ret=os.system(cmd.cmd)
|
||||
if(ret):
|
||||
return_list.put((cpu_index,False))
|
||||
return
|
||||
return_list.put((cpu_index,True))
|
||||
|
||||
|
||||
def run_cmd_queue(cmd_queue:Queue,cpu_num:int=4):
|
||||
if(cmd_queue.empty()):
|
||||
return
|
||||
process_list = []
|
||||
return_list=Queue()
|
||||
for i in range(cpu_num):
|
||||
p = Process(target=run_cmd,args=(cmd_queue,i,return_list,))
|
||||
p.start()
|
||||
process_list.append(p)
|
||||
for i in process_list:
|
||||
p.join()
|
||||
while not return_list.empty():
|
||||
i=return_list.get()
|
||||
if(not i[1]):
|
||||
print(f"子进程 [{i[0]}] 运行失败")
|
||||
exit(-1)
|
||||
|
||||
|
||||
# 保证目标都存在
|
||||
def check_exists(src:list):
|
||||
for item in src:
|
||||
for i in range(10):
|
||||
if(os.path.exists(item)):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
# 生成依赖关系
|
||||
def build_depend(src:list):
|
||||
CmdQueue=Queue()
|
||||
dst_list=[]
|
||||
flags=f"{' '.join(CINC)} {' '.join(CDEF)} {' '.join(CFLAG)}"
|
||||
for i in src:
|
||||
name=os.path.splitext(tran_path(i))[0]
|
||||
dst='.'.join([name,'d'])
|
||||
if(check_rebuild(dst,[i])):
|
||||
cmd=f"{CC} -MM {i} -o {dst} {flags}"
|
||||
print(f"更新 {dst}")
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
else:
|
||||
# print(f"{i} 没有更新源文件")
|
||||
pass
|
||||
CmdQueue.put(cmd_item_t(cmd,f"更新 {dst}"))
|
||||
dst_list.append(dst)
|
||||
run_cmd_queue(CmdQueue)
|
||||
check_exists(dst_list)
|
||||
|
||||
|
||||
# 生成中间文件
|
||||
def build_object(src:list):
|
||||
CmdQueue=Queue()
|
||||
dst_list=[]
|
||||
flags=f"{' '.join(CINC)} {' '.join(CDEF)} {' '.join(CFLAG)}"
|
||||
for i in src:
|
||||
name_t=os.path.splitext(tran_path(i))
|
||||
@@ -143,25 +183,13 @@ def build_object(src:list):
|
||||
file_type=name_t[-1]
|
||||
dst='.'.join([name,'o'])
|
||||
cd='.'.join([name,'d'])
|
||||
cmd = ''
|
||||
if(file_type in ['.c','.C']):
|
||||
if(check_rebuild(dst,read_depend_files(cd))):
|
||||
cmd=f"{CC} -c {i} -o {dst} {flags}"
|
||||
else:
|
||||
# print(f"{i} 没有更新依赖关系")
|
||||
pass
|
||||
elif(file_type in ['.s','.S','.asm','.ASM']):
|
||||
if(check_rebuild(dst,[i])):
|
||||
# cmd=f"{AS} -c {i} -o {dst} {flags}"
|
||||
pass
|
||||
else:
|
||||
# print(f"{i} 没有更新依赖关系")
|
||||
pass
|
||||
if(len(cmd)>0):
|
||||
print(f"编译 {dst}")
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
CmdQueue.put(cmd_item_t(cmd,f"编译 {dst}"))
|
||||
dst_list.append(dst)
|
||||
run_cmd_queue(CmdQueue)
|
||||
check_exists(dst_list)
|
||||
|
||||
|
||||
# 生成可执行文件
|
||||
@@ -178,9 +206,6 @@ def build_target(src:list):
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
else:
|
||||
# print(f"{dst} 没有更新的链接文件")
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
@@ -195,16 +220,17 @@ def main():
|
||||
CSRC=search_lambda(CSRC)
|
||||
moc_file_create(f"{BUILD_DIR}/moc_tmp.c",list(item[2:] for item in CINC))
|
||||
CSRC.append(f"{BUILD_DIR}/moc_tmp.c")
|
||||
# ASRC+=find_type('./',['s','S','asm','ASM'])
|
||||
|
||||
|
||||
build_depend(CSRC)
|
||||
build_object(CSRC)
|
||||
# build_object(ASRC)
|
||||
build_target(CSRC+ASRC)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
tick_start=time.time()
|
||||
main()
|
||||
tick_end=time.time()
|
||||
print(f"cost: {tick_end-tick_start}")
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import os
|
||||
import sys
|
||||
import time
|
||||
import shutil
|
||||
import dataclasses
|
||||
from multiprocessing import Process,Queue
|
||||
|
||||
|
||||
|
||||
@@ -127,24 +129,70 @@ def read_depend_files(name:str):
|
||||
return t
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class cmd_item_t:
|
||||
cmd:str
|
||||
info:str
|
||||
|
||||
def run_cmd(cmd_queue:Queue,cpu_index:int,return_list:Queue):
|
||||
while(not cmd_queue.empty()):
|
||||
try:
|
||||
cmd=cmd_queue.get_nowait()
|
||||
except Exception:
|
||||
return_list.put((cpu_index,True))
|
||||
return
|
||||
print(f"[{cpu_index}] {cmd.info}")
|
||||
ret=os.system(cmd.cmd)
|
||||
if(ret):
|
||||
return_list.put((cpu_index,False))
|
||||
return
|
||||
return_list.put((cpu_index,True))
|
||||
|
||||
|
||||
def run_cmd_queue(cmd_queue:Queue,cpu_num:int=4):
|
||||
if(cmd_queue.empty()):
|
||||
return
|
||||
process_list = []
|
||||
return_list=Queue()
|
||||
for i in range(cpu_num):
|
||||
p = Process(target=run_cmd,args=(cmd_queue,i,return_list,))
|
||||
p.start()
|
||||
process_list.append(p)
|
||||
for i in process_list:
|
||||
p.join()
|
||||
while not return_list.empty():
|
||||
i=return_list.get()
|
||||
if(not i[1]):
|
||||
print(f"子进程 [{i[0]}] 运行失败")
|
||||
exit(-1)
|
||||
|
||||
|
||||
# 保证目标都存在
|
||||
def check_exists(src:list):
|
||||
for item in src:
|
||||
for i in range(10):
|
||||
if(os.path.exists(item)):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
# 生成依赖关系
|
||||
def build_depend(src:list):
|
||||
CmdQueue=Queue()
|
||||
dst_list=[]
|
||||
flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
|
||||
for i in src:
|
||||
name=os.path.splitext(tran_path(i))[0]
|
||||
dst='.'.join([name,'d'])
|
||||
if(check_rebuild(dst,[i])):
|
||||
cmd=f"{CC} -MM {i} -o {dst} {flags}"
|
||||
print(f"更新 {dst}")
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
else:
|
||||
# print(f"{i} 没有更新源文件")
|
||||
pass
|
||||
CmdQueue.put(cmd_item_t(cmd,f"更新 {dst}"))
|
||||
dst_list.append(dst)
|
||||
run_cmd_queue(CmdQueue)
|
||||
check_exists(dst_list)
|
||||
|
||||
# 生成中间文件
|
||||
def build_object(src:list):
|
||||
CmdQueue=Queue()
|
||||
dst_list=[]
|
||||
flags=f"{' '.join(INC)} {' '.join(DEF)} {' '.join(CFLAG)}"
|
||||
for i in src:
|
||||
name_t=os.path.splitext(tran_path(i))
|
||||
@@ -156,20 +204,14 @@ def build_object(src:list):
|
||||
if(file_type in ['.c','.C']):
|
||||
if(check_rebuild(dst,read_depend_files(cd))):
|
||||
cmd=f"{CC} -c {i} -o {dst} {flags}"
|
||||
else:
|
||||
# print(f"{i} 没有更新依赖关系")
|
||||
pass
|
||||
elif(file_type in ['.s','.S','.asm','.ASM']):
|
||||
if(check_rebuild(dst,read_depend_files(cd))):
|
||||
cmd=f"{CC} -c {i} -o {dst} {flags}"
|
||||
else:
|
||||
# print(f"{i} 没有更新依赖关系")
|
||||
pass
|
||||
if(len(cmd)>0):
|
||||
print(f"编译 {dst}")
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
CmdQueue.put(cmd_item_t(cmd,f"编译 {dst}"))
|
||||
dst_list.append(dst)
|
||||
run_cmd_queue(CmdQueue)
|
||||
check_exists(dst_list)
|
||||
|
||||
|
||||
# 生成可执行文件
|
||||
@@ -186,9 +228,6 @@ def build_target(src:list):
|
||||
ret=os.system(cmd)
|
||||
if(ret):
|
||||
exit()
|
||||
else:
|
||||
# print(f"{dst} 没有更新的链接文件")
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -208,4 +247,7 @@ def main():
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tick_start=time.time()
|
||||
main()
|
||||
tick_end=time.time()
|
||||
print(f"cost: {tick_end-tick_start}")
|
||||
|
||||
Reference in New Issue
Block a user