|
|
@ -251,18 +251,19 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
|
|
|
|
in_queue = Queue(buffer_size)
|
|
|
|
in_queue = Queue(buffer_size)
|
|
|
|
out_queue = Queue(buffer_size)
|
|
|
|
out_queue = Queue(buffer_size)
|
|
|
|
out_order = [0]
|
|
|
|
out_order = [0]
|
|
|
|
|
|
|
|
|
|
|
|
# define a worker to read samples from reader to in_queue
|
|
|
|
# define a worker to read samples from reader to in_queue
|
|
|
|
def read_worker(reader, in_queue):
|
|
|
|
def read_worker(reader, in_queue):
|
|
|
|
for i in reader():
|
|
|
|
for i in reader():
|
|
|
|
in_queue.put(i)
|
|
|
|
in_queue.put(i)
|
|
|
|
in_queue.put(end)
|
|
|
|
in_queue.put(end)
|
|
|
|
|
|
|
|
|
|
|
|
# define a worker to read samples from reader to in_queue with order flag
|
|
|
|
# define a worker to read samples from reader to in_queue with order flag
|
|
|
|
def order_read_worker(reader, in_queue):
|
|
|
|
def order_read_worker(reader, in_queue):
|
|
|
|
in_order = 0
|
|
|
|
in_order = 0
|
|
|
|
for i in reader():
|
|
|
|
for i in reader():
|
|
|
|
in_queue.put((in_order,i))
|
|
|
|
in_queue.put((in_order, i))
|
|
|
|
in_order+=1
|
|
|
|
in_order += 1
|
|
|
|
in_queue.put(end)
|
|
|
|
in_queue.put(end)
|
|
|
|
|
|
|
|
|
|
|
|
# start a read worker in a thread
|
|
|
|
# start a read worker in a thread
|
|
|
@ -281,7 +282,7 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
|
|
|
|
sample = in_queue.get()
|
|
|
|
sample = in_queue.get()
|
|
|
|
in_queue.put(end)
|
|
|
|
in_queue.put(end)
|
|
|
|
out_queue.put(end)
|
|
|
|
out_queue.put(end)
|
|
|
|
|
|
|
|
|
|
|
|
# define a worker to handle samples from in_queue by mapper
|
|
|
|
# define a worker to handle samples from in_queue by mapper
|
|
|
|
# and put mapped samples into out_queue by order
|
|
|
|
# and put mapped samples into out_queue by order
|
|
|
|
def order_handle_worker(in_queue, out_queue, mapper, out_order):
|
|
|
|
def order_handle_worker(in_queue, out_queue, mapper, out_order):
|
|
|
@ -292,18 +293,18 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
|
|
|
|
while order != out_order[0]:
|
|
|
|
while order != out_order[0]:
|
|
|
|
pass
|
|
|
|
pass
|
|
|
|
out_queue.put(r)
|
|
|
|
out_queue.put(r)
|
|
|
|
out_order[0] += 1
|
|
|
|
out_order[0] += 1
|
|
|
|
ins = in_queue.get()
|
|
|
|
ins = in_queue.get()
|
|
|
|
in_queue.put(end)
|
|
|
|
in_queue.put(end)
|
|
|
|
out_queue.put(end)
|
|
|
|
out_queue.put(end)
|
|
|
|
|
|
|
|
|
|
|
|
# start several handle_workers
|
|
|
|
# start several handle_workers
|
|
|
|
target = order_handle_worker if order else handle_worker
|
|
|
|
target = order_handle_worker if order else handle_worker
|
|
|
|
args = (in_queue, out_queue, mapper, out_order) if order else (in_queue, out_queue, mapper)
|
|
|
|
args = (in_queue, out_queue, mapper, out_order) if order else (
|
|
|
|
|
|
|
|
in_queue, out_queue, mapper)
|
|
|
|
workers = []
|
|
|
|
workers = []
|
|
|
|
for i in xrange(process_num):
|
|
|
|
for i in xrange(process_num):
|
|
|
|
worker = Thread(
|
|
|
|
worker = Thread(target=target, args=args)
|
|
|
|
target=target, args=args)
|
|
|
|
|
|
|
|
worker.daemon = True
|
|
|
|
worker.daemon = True
|
|
|
|
workers.append(worker)
|
|
|
|
workers.append(worker)
|
|
|
|
for w in workers:
|
|
|
|
for w in workers:
|
|
|
|