Joblib: Python 3.7, Windows: parallelisation does not work

Created on 28 Jun 2019 · 10Comments · Source: joblib/joblib

I'm currently tracking down why I can't get n_jobs=-1 to work for something else, and found the following when trying the examples:

>>> from joblib import Parallel, delayed
>>> from math import modf
>>> Parallel(n_jobs=2)(delayed(modf)(i/2.) for i in range(10))

exception calling callback for <Future at 0x2d4dd8b97b8 state=finished raised BrokenProcessPool>
joblib.externals.loky.process_executor._RemoteTraceback:
'''
Traceback (most recent call last):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\process_executor.py", line 391, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "c:\program files\python37\lib\multiprocessing\queues.py", line 99, in get
    if not self._rlock.acquire(block, timeout):
PermissionError: [WinError 5] Access is denied
'''

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\_base.py", line 625, in _invoke_callbacks
    callback(self)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 309, in __call__
    self.parallel.dispatch_next()
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 731, in dispatch_next
    if not self.dispatch_one_batch(self._original_iterator):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 759, in dispatch_one_batch
    self._dispatch(tasks)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 716, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\_parallel_backends.py", line 510, in apply_async
    future = self._workers.submit(SafeFunction(func))
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\reusable_executor.py", line 151, in submit
    fn, *args, **kwargs)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\process_executor.py", line 1022, in submit
    raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. 
Please ensure that the arguments of the function are all picklable.
ERROR: The process with PID 11044 (child process of PID 10084) could not be terminated.
Reason: There is no running instance of the task.
ERROR: The process with PID 10084 (child process of PID 10936) could not be terminated.
Reason: There is no running instance of the task.
ERROR: The process "10020" not found.
joblib.externals.loky.process_executor._RemoteTraceback: 
'''
Traceback (most recent call last):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\process_executor.py", line 391, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "c:\program files\python37\lib\multiprocessing\queues.py", line 99, in get
    if not self._rlock.acquire(block, timeout):
PermissionError: [WinError 5] Access is denied
'''

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 934, in __call__
    self.retrieve()
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 833, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\_parallel_backends.py", line 521, in wrap_future_result
    return future.result(timeout=timeout)
  File "c:\program files\python37\lib\concurrent\futures\_base.py", line 432, in result      
    return self.__get_result()
  File "c:\program files\python37\lib\concurrent\futures\_base.py", line 384, in __get_result
    raise self._exception
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\_base.py", line 625, in _invoke_callbacks
    callback(self)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 309, in __call__
    self.parallel.dispatch_next()
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 731, in dispatch_next
    if not self.dispatch_one_batch(self._original_iterator):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 759, in dispatch_one_batch
    self._dispatch(tasks)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 716, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\_parallel_backends.py", line 510, in apply_async
    future = self._workers.submit(SafeFunction(func))
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\reusable_executor.py", line 151, in submit
    fn, *args, **kwargs)
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\process_executor.py", line 1022, in submit
    raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. 
Please ensure that the arguments of the function are all picklable.

The errors may vary:

>>> Parallel(n_jobs=2, verbose=10)(delayed(modf)(i/2.) for i in range(10)) 
[Parallel(n_jobs=2)]: Using backend LokyBackend with 2 concurrent workers.
[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:    0.4s
[Parallel(n_jobs=2)]: Done   2 tasks      | elapsed:    0.4s
[Parallel(n_jobs=2)]: Done   3 tasks      | elapsed:    0.4s
[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:    0.4s
ERROR: The process "6640" not found.
joblib.externals.loky.process_executor._RemoteTraceback: 
'''
Traceback (most recent call last):
  File "c:\program files\python37\lib\multiprocessing\queues.py", line 109, in get
    self._sem.release()
OSError: [WinError 6] The handle is invalid

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\externals\loky\process_executor.py", line 391, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "c:\program files\python37\lib\multiprocessing\queues.py", line 111, in get
    self._rlock.release()
OSError: [WinError 6] The handle is invalid
'''

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 934, in __call__
    self.retrieve()
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\parallel.py", line 833, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "C:\UsersLocal\nikolaus.waxweiler\Envs\damakerngrouper-py3.7\lib\site-packages\joblib\_parallel_backends.py", line 521, in wrap_future_result
    return future.result(timeout=timeout)
  File "c:\program files\python37\lib\concurrent\futures\_base.py", line 432, in result      
    return self.__get_result()
  File "c:\program files\python37\lib\concurrent\futures\_base.py", line 384, in __get_result
    raise self._exception
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. 
Please ensure that the arguments of the function are all picklable.

Python 3.7.3 x64 on Windows 10 1903, joblib 0.13.2.

windows

Source

madig

All 10 comments

It works on 3.6.8 x64 (same Windows version) and on Ubuntu 18.04 x64 with Python 3.6.8 and 3.7.3, so this is maybe a Python 3.7.x for Windows bug?

madig on 28 Jun 2019

That looks like a low level problem. Maybe @pierreglaser and @tomMoral have an idea.

ogrisel on 5 Jul 2019

Also we have windows CI for loky so this is weird we did not observe this crash ourself.

ogrisel on 5 Jul 2019

Still happens on 3.7.4.

Edit: I remember similar error messages when using python-doit in multiprocessing mode on 3.7 for Windows. Maybe it really is a low-level issue with multiprocessing in 3.7 for Windows.

madig on 24 Jul 2019

Hi @madig. I can reproduce. Will look at it ASAP.

pierreglaser on 25 Jul 2019

🎉1

@madig I implemeted a fix in tomMoral/loky#216, which hopefully will be included in the next joblib release.

pierreglaser on 9 Sep 2019

🎉1

Wow, that is an insidious one. Thanks for your work!

madig on 9 Sep 2019

This fix is included in joblib 0.14.0.

ogrisel on 6 Dec 2019

🚀1

Thanks!

madig on 6 Dec 2019

I'm having the same error as reported here although I'm using joblib 0.15.1.

The code that throws the error:

from joblib import Parallel, delayed
from math import modf
Parallel(n_jobs=2)(delayed(modf)(i/2.) for i in range(10))

Complete error trace:

exception calling callback for <Future at 0x247aeae0cc0 state=finished raised BrokenProcessPool>
joblib.externals.loky.process_executor._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\multiprocessing\queues.py", line 109, in get
    self._sem.release()
OSError: [WinError 6] The handle is invalid

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\process_executor.py", line 404, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\multiprocessing\queues.py", line 111, in get
    self._rlock.release()
OSError: [WinError 6] The handle is invalid
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\_base.py", line 625, in _invoke_callbacks
    callback(self)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 347, in __call__
    self.parallel.dispatch_next()
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 780, in dispatch_next
    if not self.dispatch_one_batch(self._original_iterator):
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 847, in dispatch_one_batch
    self._dispatch(tasks)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 765, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\_parallel_backends.py", line 529, in apply_async
    future = self._workers.submit(SafeFunction(func))
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\reusable_executor.py", line 178, in submit
    fn, *args, **kwargs)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\process_executor.py", line 1102, in submit
    raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.
ERROR: The process "15156" not found.
ERROR: The process "16604" not found.
joblib.externals.loky.process_executor._RemoteTraceback: 
"""
Traceback (most recent call last):
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\multiprocessing\queues.py", line 109, in get
    self._sem.release()
OSError: [WinError 6] The handle is invalid

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\process_executor.py", line 404, in _process_worker
    call_item = call_queue.get(block=True, timeout=timeout)
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\multiprocessing\queues.py", line 111, in get
    self._rlock.release()
OSError: [WinError 6] The handle is invalid
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:/Users/arpada/Wok/Projects/2020/KIP KpNK Quantifying reliability/prob_taralli/test/loglikelihood_builder/test_multiprocessing.py", line 8, in <module>
    Parallel(n_jobs=2)(delayed(modf)(i/2.) for i in range(10))
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 1042, in __call__
    self.retrieve()
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 921, in retrieve
    self._output.extend(job.get(timeout=self.timeout))
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\_parallel_backends.py", line 540, in wrap_future_result
    return future.result(timeout=timeout)
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\concurrent\futures\_base.py", line 432, in result
    return self.__get_result()
  File "c:\Users\arpada\AppData\Local\Programs\Python\Python37\lib\concurrent\futures\_base.py", line 384, in __get_result
    raise self._exception
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\_base.py", line 625, in _invoke_callbacks
    callback(self)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 347, in __call__
    self.parallel.dispatch_next()
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 780, in dispatch_next
    if not self.dispatch_one_batch(self._original_iterator):
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 847, in dispatch_one_batch
    self._dispatch(tasks)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\parallel.py", line 765, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\_parallel_backends.py", line 529, in apply_async
    future = self._workers.submit(SafeFunction(func))
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\reusable_executor.py", line 178, in submit
    fn, *args, **kwargs)
  File "C:\Users\arpada\Wok\Projects\2020\KIP KpNK Quantifying reliability\prob_taralli\venv\lib\site-packages\joblib\externals\loky\process_executor.py", line 1102, in submit
    raise self._flags.broken
joblib.externals.loky.process_executor.BrokenProcessPool: A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.

Process finished with exit code 1