1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 from contextlib import contextmanager
24 from threading import Lock
25 import Queue as Q
26 import gtk
27 import os
28 import threading as td
29 import time
32 """
33 A class of thread pool.
34
35 @undocumented: loop
36 @undocumented: start_missions
37 @undocumented: wake_up_wait_missions
38 @undocumented: sync
39 @undocumented: finish_mission
40 @undocumented: clean_mission
41 """
42
43 FINISH_SIGNAL = "Finish"
44
45 - def __init__(self,
46 concurrent_thread_num=5,
47 clean_delay=0,
48 clean_callback=None,
49 exit_when_finish=False
50 ):
51 """
52 Initialise the thread pool.
53
54 @param concurrent_thread_num: Max concurrent thread number.
55 @param clean_delay: The time between the finish of the thread and the invocation of thread clean up function.
56 @param clean_callback: The clean up function, which is invoked after the thread is finished.
57 @param exit_when_finish: Indicates whether the thread pool should be destroyed after all mission is finished. By default, it's False.
58 """
59
60 td.Thread.__init__(self)
61 self.setDaemon(True)
62
63
64 self.concurrent_thread_num = concurrent_thread_num
65 self.clean_delay = clean_delay
66 self.clean_callback = clean_callback
67 self.clean_time = time.time()
68 self.exit_when_finish = exit_when_finish
69
70
71 self.active_mission_list = []
72 self.wait_mission_list = []
73 self.mission_result_list = []
74
75
76 self.thread_sync_lock = Lock()
77 self.mission_lock = Q.Queue()
78
80 """
81 The thread function.
82 """
83 self.loop()
84
86 """
87 Internal function to do loop.
88
89 It is called recusivly to start new mission and monitor the missions' status.
90 """
91 result = self.mission_lock.get()
92 if result == self.FINISH_SIGNAL:
93 print ">>> Finish missions."
94 if self.exit_when_finish:
95 print ">>> Exit thread pool %s" % (self)
96 else:
97 print ">>> Wait new missions."
98 self.loop()
99 else:
100 self.start_missions(result)
101 self.loop()
102
104 """
105 Add missions to the thread pool.
106
107 @param missions: A list of mission which is of type class MissionThread.
108 """
109 self.mission_lock.put(missions)
110
112 """
113 Internal function to start missions in the thread pool.
114
115 @param missions: A list of mission which is of type class MissionThread.
116 """
117 for (index, mission) in enumerate(missions):
118
119 if len(self.active_mission_list) >= self.concurrent_thread_num:
120 self.wait_mission_list += missions[index::]
121 break
122
123 else:
124 self.start_mission(mission)
125
127 """
128 Start a specific mission in the thread pool.
129
130 @param mission: a mission which is of type class MissionThread.
131 """
132 self.active_mission_list.append(mission)
133 mission.finish_mission = self.finish_mission
134 mission.start()
135
137 """
138 Internal function to wake up the mission which is in waiting list.
139 """
140 for mission in self.wait_mission_list:
141
142 if len(self.active_mission_list) >= self.concurrent_thread_num:
143 break
144
145 else:
146
147 if mission in self.wait_mission_list:
148 self.wait_mission_list.remove(mission)
149
150
151 self.start_mission(mission)
152
153 @contextmanager
155 """
156 Internal function do synchronize jobs.
157 """
158 self.thread_sync_lock.acquire()
159 try:
160 yield
161 except Exception, e:
162 print 'sync error %s' % e
163 else:
164 self.thread_sync_lock.release()
165
167 """
168 Internal function that invoked by the MissionThread for every certain time.
169
170 @param mission: A mission of type MissionThread.
171 """
172 with self.sync():
173
174 if mission in self.active_mission_list:
175 self.mission_result_list.append(mission.get_mission_result())
176 self.active_mission_list.remove(mission)
177
178
179 self.wake_up_wait_missions()
180
181
182 if self.clean_delay > 0 and self.clean_callback != None and len(self.mission_result_list) > 0:
183
184 current_time = time.time()
185
186
187 if (len(self.active_mission_list) == 0 and len(self.wait_mission_list) == 0) or (current_time - self.clean_time) * 1000 > self.clean_delay:
188 self.clean_mission(current_time)
189
190
191 if (len(self.active_mission_list) == 0 and len(self.wait_mission_list) == 0):
192 self.mission_lock.put(self.FINISH_SIGNAL)
193
195 """
196 Internal function to call clean_callback of the mission which is invoked in finish_mission.
197
198 @param current_time: the time of type float which indicates the time the clean_mission is invoked.
199 """
200
201 self.clean_callback(self.mission_result_list)
202
203
204 self.mission_result_list = []
205
206
207 self.clean_time = current_time
208
210 """
211 This class stands for a single mission in the thread pool.
212 """
213
215 """
216 Initialise the MissionThread.
217 """
218 td.Thread.__init__(self)
219 self.setDaemon(True)
220
222 """
223 The thread function.
224 """
225 self.start_mission()
226 self.finish_mission(self)
227
229 """
230 The mission thread function of MissionThread.
231
232 This function is MissionThread template, you should write your own implementation.
233 """
234 print "Write your code here."
235
237 """
238 Return the mission result.
239
240 This function is MissionThread template, you should write your own implementation.
241
242 @return: If you don't want handle result, just return None.
243 """
244 return None
245
247 '''Test mission thread.'''
248
253
255 '''Start misssion.'''
256 print "*** Start download cover for %s" % (self.artist)
257 time.sleep(5)
258 print "*** Finish download cover for %s" % (self.artist)
259
261 '''Get misssion retsult.'''
262 return os.path.join("/home/cover", self.artist)
263
265 '''Clean cover.'''
266 print "#### Update covers %s" % (filepath)
267
268 if __name__ == "__main__":
269 gtk.gdk.threads_init()
270
271 artists_one = ["John Lennon", "Aqua", "Beatles", "Bob Dylan", "BSB", "Anastacia", "Coldplay",
272 "James Blunt", "James Morrison", "Jason Mraz"]
273 artists_two = ["Daniel Powter", "Dixie Chicks", "Egil Olsen", "Elton John", "Elvis Presley"]
274
275 missions_one = []
276 for artist in artists_one:
277 missions_one.append(TestMissionThread(artist))
278
279 missions_two = []
280 for artist in artists_two:
281 missions_two.append(TestMissionThread(artist))
282
283 pool = MissionThreadPool(5, 1000, clean_cover)
284 pool.start()
285 pool.add_missions(missions_one)
286 pool.add_missions(missions_two)
287
288 gtk.main()
289