Open
Description
Hello, I recently paid attention to your part of this tutorial, but I encountered source code problems when implementing your tutorial, when I executed
from ktext.preprocess import processor
body_pp = processor(keep_n=8000, padding_maxlen=70)
train_body_vecs = body_pp.fit_transform(train_body_raw)
I got the following error:
OSError Traceback (most recent call last)
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/ktext/preprocess.py in apply_parallel(func, data, cpu_cores)
72 chunk_size = ceil(len(data) / cpu_cores)
---> 73 pool = Pool(cpu_cores)
74 transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/pool.py in __init__(self, processes, initializer, initargs, maxtasksperchild, context)
173 self._pool = []
--> 174 self._repopulate_pool()
175
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/pool.py in _repopulate_pool(self)
238 w.daemon = True
--> 239 w.start()
240 util.debug('added worker')
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/process.py in start(self)
104 _cleanup()
--> 105 self._popen = self._Popen(self)
106 self._sentinel = self._popen.sentinel
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/context.py in _Popen(process_obj)
276 from .popen_fork import Popen
--> 277 return Popen(process_obj)
278
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/popen_fork.py in __init__(self, process_obj)
18 self.returncode = None
---> 19 self._launch(process_obj)
20
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/multiprocess/popen_fork.py in _launch(self, process_obj)
65 parent_r, child_w = os.pipe()
---> 66 self.pid = os.fork()
67 if self.pid == 0:
OSError: [Errno 12] Cannot allocate memory
During handling of the above exception, another exception occurred:
UnboundLocalError Traceback (most recent call last)
<timed exec> in <module>()
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/ktext/preprocess.py in fit_transform(self, data)
336
337 """
--> 338 tokenized_data = self.fit(data, return_tokenized_data=True)
339
340 logging.warning(f'...fit is finished, beginning transform')
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/ktext/preprocess.py in fit(self, data, return_tokenized_data)
278 now = get_time()
279 logging.warning(f'....tokenizing data')
--> 280 tokenized_data = self.parallel_process_text(data)
281
282 if not self.padding_maxlen:
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/ktext/preprocess.py in parallel_process_text(self, data)
233 end_tok=self.end_tok)
234 n_cores = self.num_cores
--> 235 return flattenlist(apply_parallel(process_text, data, n_cores))
236
237 def generate_doc_length_stats(self):
~/anaconda3/envs/deep2.0.0/lib/python3.6/site-packages/ktext/preprocess.py in apply_parallel(func, data, cpu_cores)
74 transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
75 finally:
---> 76 pool.close()
77 pool.join()
78 return transformed_data
UnboundLocalError: local variable 'pool' referenced before assignment
This error seems to be due to an error caused by your source code. Can you see what happened?
Metadata
Metadata
Assignees
Labels
No labels