1. 程式人生 > 程式設計 >Python3如何使用多執行緒升程式執行速度

Python3如何使用多執行緒升程式執行速度

優化前後新老程式碼如下:

from git_tools.git_tool import get_collect_projects,QQNews_Git
from threading import Thread,Lock
import datetime

base_url = "http://git.xx.com"
project_members_commits_lang_info = {}
lock = Lock()
threads = []

'''
Author:zenkilan
'''


def count_time(func):
  def took_up_time(*args,**kwargs):
    start_time = datetime.datetime.now()
    ret = func(*args,**kwargs)
    end_time = datetime.datetime.now()
    took_up_time = (end_time - start_time).total_seconds()
    print(f"{func.__name__} execution took up time:{took_up_time}")
    return ret

  return took_up_time


def get_project_member_lang_code_lines(git,member,begin_date,end_date):
  global project_members_commits_lang_info
  global lock
  member_name = member["username"]
  r = git.get_user_info(member_name)
  if not r["id"]:
    return
  user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"],end_date)
  if len(user_commits_lang_info) == 0:
    return
  lock.acquire()
  project_members_commits_lang_info.setdefault(git.project,dict())
  project_members_commits_lang_info[git.project][member_name] = user_commits_lang_info
  lock.release()


def get_project_lang_code_lines(project,end_date):
  global threads
  git = QQNews_Git(project[1],base_url,project[0])
  project_members = git.get_project_members()
  if len(project_members) == 0:
    return
  for member in project_members:
    thread = Thread(target=get_project_member_lang_code_lines,args=(git,end_date))
    threads.append(thread)
    thread.start()


@count_time
def get_projects_lang_code_lines(begin_date,end_date):
  """
  獲取專案程式碼行語言相關統計——新方法(提升效率)
  應用多執行緒替代for迴圈
  併發訪問共享外部資源
  :return:
  """
  global project_members_commits_lang_info
  global threads
  for project in get_collect_projects():
    thread = Thread(target=get_project_lang_code_lines,args=(project,end_date))
    threads.append(thread)
    thread.start()


@count_time
def get_projects_lang_code_lines_old(begin_date,end_date):
  """
  獲取專案程式碼行語言相關統計——老方法(耗時嚴重)
  使用最基本的思路進行程式設計
  雙層for迴圈巢狀並且每層都包含耗時操作
  :return:
  """
  project_members_commits_lang_info = {}
  for project in get_collect_projects():
    git = QQNews_Git(project[1],project[0])
    project_members = git.get_project_members()
    user_commits_lang_info_dict = {}
    if len(project_members) == 0:
      continue
    for member in project_members:
      member_name = member["username"]
      r = git.get_user_info(member_name,debug=False)
      if not r["id"]:
        continue
      try:
        user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"],end_date)
        if len(user_commits_lang_info) == 0:
          continue
        user_commits_lang_info_dict[member_name] = user_commits_lang_info
        project_members_commits_lang_info[git.project] = user_commits_lang_info_dict
      except:
        pass
  return project_members_commits_lang_info


def test_results_equal(resultA,resultB):
  """
  測試方法
  :param resultA:
  :param resultB:
  :return:
  """
  print(resultA)
  print(resultB)
  assert len(str(resultA)) == len(str(resultB))


if __name__ == '__main__':
  from git_tools.config import begin_date,end_date

  get_projects_lang_code_lines(begin_date,end_date)
  for t in threads:
    t.join()
  old_result = get_projects_lang_code_lines_old(begin_date,end_date)
  test_results_equal(old_result,project_members_commits_lang_info)

老方法裡外層for迴圈和內層for迴圈裡均存在耗時操作:

1)git.get_project_members()

2)git.get_user_info(member_name,debug=False)

分兩步來優化,先裡後外或先外后里都行。用多執行緒替換for迴圈,併發共享外部資源,加鎖避免寫衝突。

測試結果通過,函式執行時間裝飾器顯示(單位秒):

get_projects_lang_code_lines execution took up time:1.85294

get_projects_lang_code_lines_old execution took up time:108.604177

速度提升了約58倍

以上就是本文的全部內容,希望對大家的學習有所幫助,也希望大家多多支援我們。