cantor added the comment:
in python 2.7.3 this kind of works however it is less efficient than the pure
lzma.compress()
from threading import Thread
from backports import lzma
from functools import partial
import multiprocessing
class CompressClass(Thread):
def __init__ (self,data,c):
Thread.__init__(self)
self.exception=False
self.data=data
self.datacompressed=""
self.c=c
def getException(self):
return self.exception
def getOutput(self):
return self.datacompressed
def run(self):
self.datacompressed=(self.c).compress(self.data)
def split_len(seq, length):
return [seq[i:i+length] for i in range(0, len(seq), length)]
def launch_multiple_lzma(data,c):
print 'cores'
present=CompressClass(data,c)
present.start()
present.join()
return present.getOutput()
def threaded_lzma_map(sequence,threads):
lzc = lzma.LZMACompressor()
blocksize = int(round(len(sequence)/threads))
lzc_partial = partial(launch_multiple_lzma,c=lzc)
lzlist = map(lzc_partial,split_len(sequence, blocksize))
#pool=multiprocessing.Pool()
#lzclist = pool.map(lzc_partial,split_len(sequence, blocksize))
#pool.close()
#pool.join()
out_flush = lzc.flush()
res = "".join(lzlist + [out_flush])
return res
sequence = 'AAAAAJKDDDDDDDDDDDDDDDDDDDDDDDDDDDDGJFKSHFKLHALWEHAIHWEOIAH
IOAHIOWEHIOHEIOFEAFEASFEAFWEWWWWWWWWWWWWWWWWWWWWWWWWWWWWWEWFQWEWQWQGEWQFEWFDWEWEGEFGWEG'
lzma.compress(sequence) == threaded_lzma_map(sequence,threads=16)
Any way this could be imporved?
----------
_______________________________________
Python tracker <[email protected]>
<http://bugs.python.org/issue19395>
_______________________________________
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
https://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
