|
|
|
@ -251,6 +251,7 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
|
|
|
|
|
in_queue = Queue(buffer_size)
|
|
|
|
|
out_queue = Queue(buffer_size)
|
|
|
|
|
out_order = [0]
|
|
|
|
|
|
|
|
|
|
# define a worker to read samples from reader to in_queue
|
|
|
|
|
def read_worker(reader, in_queue):
|
|
|
|
|
for i in reader():
|
|
|
|
@ -299,11 +300,11 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
|
|
|
|
|
|
|
|
|
|
# start several handle_workers
|
|
|
|
|
target = order_handle_worker if order else handle_worker
|
|
|
|
|
args = (in_queue, out_queue, mapper, out_order) if order else (in_queue, out_queue, mapper)
|
|
|
|
|
args = (in_queue, out_queue, mapper, out_order) if order else (
|
|
|
|
|
in_queue, out_queue, mapper)
|
|
|
|
|
workers = []
|
|
|
|
|
for i in xrange(process_num):
|
|
|
|
|
worker = Thread(
|
|
|
|
|
target=target, args=args)
|
|
|
|
|
worker = Thread(target=target, args=args)
|
|
|
|
|
worker.daemon = True
|
|
|
|
|
workers.append(worker)
|
|
|
|
|
for w in workers:
|
|
|
|
|