Package dtk :: Package ui :: Module thread_pool

Source Code for Module dtk.ui.thread_pool

  1  #! /usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  # Copyright (C) 2011 ~ 2012 Deepin, Inc. 
  5  #               2011 ~ 2012 Wang Yong 
  6  #  
  7  # Author:     Wang Yong <lazycat.manatee@gmail.com> 
  8  # Maintainer: Wang Yong <lazycat.manatee@gmail.com> 
  9  #  
 10  # This program is free software: you can redistribute it and/or modify 
 11  # it under the terms of the GNU General Public License as published by 
 12  # the Free Software Foundation, either version 3 of the License, or 
 13  # any later version. 
 14  #  
 15  # This program is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 18  # GNU General Public License for more details. 
 19  #  
 20  # You should have received a copy of the GNU General Public License 
 21  # along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 22   
 23  from contextlib import contextmanager  
 24  from threading import Lock 
 25  import Queue as Q 
 26  import gtk 
 27  import os 
 28  import threading as td 
 29  import time 
30 31 -class MissionThreadPool(td.Thread):
32 """ 33 A class of thread pool. 34 35 @undocumented: loop 36 @undocumented: start_missions 37 @undocumented: wake_up_wait_missions 38 @undocumented: sync 39 @undocumented: finish_mission 40 @undocumented: clean_mission 41 """ 42 43 FINISH_SIGNAL = "Finish" 44
45 - def __init__(self, 46 concurrent_thread_num=5, # max concurrent thread number 47 clean_delay=0, # clean delay (milliseconds) 48 clean_callback=None, # clean callback 49 exit_when_finish=False # exit thread pool when all missions finish 50 ):
51 """ 52 Initialise the thread pool. 53 54 @param concurrent_thread_num: Max concurrent thread number. 55 @param clean_delay: The time between the finish of the thread and the invocation of thread clean up function. 56 @param clean_callback: The clean up function, which is invoked after the thread is finished. 57 @param exit_when_finish: Indicates whether the thread pool should be destroyed after all mission is finished. By default, it's False. 58 """ 59 # Init thread. 60 td.Thread.__init__(self) 61 self.setDaemon(True) # make thread exit when main program exit 62 63 # Init arguments. 64 self.concurrent_thread_num = concurrent_thread_num 65 self.clean_delay = clean_delay 66 self.clean_callback = clean_callback 67 self.clean_time = time.time() 68 self.exit_when_finish = exit_when_finish 69 70 # Init list. 71 self.active_mission_list = [] 72 self.wait_mission_list = [] 73 self.mission_result_list = [] 74 75 # Init lock. 76 self.thread_sync_lock = Lock() 77 self.mission_lock = Q.Queue()
78
79 - def run(self):
80 """ 81 The thread function. 82 """ 83 self.loop()
84
85 - def loop(self):
86 """ 87 Internal function to do loop. 88 89 It is called recusivly to start new mission and monitor the missions' status. 90 """ 91 result = self.mission_lock.get() 92 if result == self.FINISH_SIGNAL: 93 print ">>> Finish missions." 94 if self.exit_when_finish: 95 print ">>> Exit thread pool %s" % (self) 96 else: 97 print ">>> Wait new missions." 98 self.loop() 99 else: 100 self.start_missions(result) 101 self.loop()
102
103 - def add_missions(self, missions):
104 """ 105 Add missions to the thread pool. 106 107 @param missions: A list of mission which is of type class MissionThread. 108 """ 109 self.mission_lock.put(missions)
110
111 - def start_missions(self, missions):
112 """ 113 Internal function to start missions in the thread pool. 114 115 @param missions: A list of mission which is of type class MissionThread. 116 """ 117 for (index, mission) in enumerate(missions): 118 # Add to wait list if active mission number reach max value. 119 if len(self.active_mission_list) >= self.concurrent_thread_num: 120 self.wait_mission_list += missions[index::] 121 break 122 # Otherwise start new mission. 123 else: 124 self.start_mission(mission)
125
126 - def start_mission(self, mission):
127 """ 128 Start a specific mission in the thread pool. 129 130 @param mission: a mission which is of type class MissionThread. 131 """ 132 self.active_mission_list.append(mission) 133 mission.finish_mission = self.finish_mission 134 mission.start()
135
136 - def wake_up_wait_missions(self):
137 """ 138 Internal function to wake up the mission which is in waiting list. 139 """ 140 for mission in self.wait_mission_list: 141 # Just break loop when active mission is bigger than max value. 142 if len(self.active_mission_list) >= self.concurrent_thread_num: 143 break 144 # Otherwise add mission from wait list. 145 else: 146 # Remove from wait list. 147 if mission in self.wait_mission_list: 148 self.wait_mission_list.remove(mission) 149 150 # Start new mission. 151 self.start_mission(mission)
152 153 @contextmanager
154 - def sync(self):
155 """ 156 Internal function do synchronize jobs. 157 """ 158 self.thread_sync_lock.acquire() 159 try: 160 yield 161 except Exception, e: 162 print 'sync error %s' % e 163 else: 164 self.thread_sync_lock.release()
165
166 - def finish_mission(self, mission):
167 """ 168 Internal function that invoked by the MissionThread for every certain time. 169 170 @param mission: A mission of type MissionThread. 171 """ 172 with self.sync(): 173 # Remove mission from active mission list. 174 if mission in self.active_mission_list: 175 self.mission_result_list.append(mission.get_mission_result()) 176 self.active_mission_list.remove(mission) 177 178 # Wake up wait missions. 179 self.wake_up_wait_missions() 180 181 # Do clean work. 182 if self.clean_delay > 0 and self.clean_callback != None and len(self.mission_result_list) > 0: 183 # Get current time. 184 current_time = time.time() 185 186 # Do clean work no mission will start or time reach delay. 187 if (len(self.active_mission_list) == 0 and len(self.wait_mission_list) == 0) or (current_time - self.clean_time) * 1000 > self.clean_delay: 188 self.clean_mission(current_time) 189 190 # Exit thread when download finish. 191 if (len(self.active_mission_list) == 0 and len(self.wait_mission_list) == 0): 192 self.mission_lock.put(self.FINISH_SIGNAL)
193
194 - def clean_mission(self, current_time):
195 """ 196 Internal function to call clean_callback of the mission which is invoked in finish_mission. 197 198 @param current_time: the time of type float which indicates the time the clean_mission is invoked. 199 """ 200 # Do clean work. 201 self.clean_callback(self.mission_result_list) 202 203 # Clean mission result list. 204 self.mission_result_list = [] 205 206 # Record new clean time. 207 self.clean_time = current_time
208
209 -class MissionThread(td.Thread):
210 """ 211 This class stands for a single mission in the thread pool. 212 """ 213
214 - def __init__(self):
215 """ 216 Initialise the MissionThread. 217 """ 218 td.Thread.__init__(self) 219 self.setDaemon(True) # make thread exit when main program exit
220
221 - def run(self):
222 """ 223 The thread function. 224 """ 225 self.start_mission() 226 self.finish_mission(self)
227
228 - def start_mission(self):
229 """ 230 The mission thread function of MissionThread. 231 232 This function is MissionThread template, you should write your own implementation. 233 """ 234 print "Write your code here."
235
236 - def get_mission_result(self):
237 """ 238 Return the mission result. 239 240 This function is MissionThread template, you should write your own implementation. 241 242 @return: If you don't want handle result, just return None. 243 """ 244 return None
245
246 -class TestMissionThread(MissionThread):
247 '''Test mission thread.''' 248
249 - def __init__(self, artist):
250 '''Init test mission thread.''' 251 MissionThread.__init__(self) 252 self.artist = artist
253
254 - def start_mission(self):
255 '''Start misssion.''' 256 print "*** Start download cover for %s" % (self.artist) 257 time.sleep(5) 258 print "*** Finish download cover for %s" % (self.artist)
259
260 - def get_mission_result(self):
261 '''Get misssion retsult.''' 262 return os.path.join("/home/cover", self.artist)
263
264 -def clean_cover(filepath):
265 '''Clean cover.''' 266 print "#### Update covers %s" % (filepath)
267 268 if __name__ == "__main__": 269 gtk.gdk.threads_init() 270 271 artists_one = ["John Lennon", "Aqua", "Beatles", "Bob Dylan", "BSB", "Anastacia", "Coldplay", 272 "James Blunt", "James Morrison", "Jason Mraz"] 273 artists_two = ["Daniel Powter", "Dixie Chicks", "Egil Olsen", "Elton John", "Elvis Presley"] 274 275 missions_one = [] 276 for artist in artists_one: 277 missions_one.append(TestMissionThread(artist)) 278 279 missions_two = [] 280 for artist in artists_two: 281 missions_two.append(TestMissionThread(artist)) 282 283 pool = MissionThreadPool(5, 1000, clean_cover) 284 pool.start() 285 pool.add_missions(missions_one) 286 pool.add_missions(missions_two) 287 288 gtk.main() 289