Skip to content

Commit 1d9c508

Browse files
tomMoraltomMoral
committed
Merge branch 'master' into FIX_freeze_support
2 parents 5bfd31e + 31b88f1 commit 1d9c508

33 files changed

+674
-285
lines changed

‎.github/workflows/test.yml

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
name: CI
2+
permissions:
3+
contents: read
4+
5+
on:
6+
push:
7+
branches:
8+
- master
9+
pull_request:
10+
branches:
11+
- master
12+
schedule:
13+
# Weekly build on Tuesday at 2:00 AM
14+
- cron: "0 2 * * 2"
15+
16+
# Cancel in-progress workflows when pushing
17+
# a new commit on the same branch
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
24+
linting:
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- uses: actions/checkout@v3
29+
- name: Set up Python
30+
uses: actions/setup-python@v4
31+
with:
32+
python-version: "3.12"
33+
- name: Install black
34+
run: |
35+
pip install black==25.1.0
36+
- name: Run black
37+
run: |
38+
black --check --diff .
39+
40+
testing:
41+
name: Testing
42+
needs: linting
43+
timeout-minutes: 30
44+
strategy:
45+
fail-fast: false
46+
matrix:
47+
include:
48+
49+
- name: windows-py313
50+
os: windows-latest
51+
PYTHON_VERSION: "3.13"
52+
- name: windows-py39
53+
os: windows-latest
54+
PYTHON_VERSION: "3.9"
55+
LOKY_TEST_NO_CIM: "true"
56+
57+
- name: macos-py313
58+
os: macos-latest
59+
PYTHON_VERSION: "3.13"
60+
- name: macos-py39
61+
os: macos-latest
62+
PYTHON_VERSION: "3.9"
63+
64+
- name: linux-py313
65+
os: ubuntu-latest
66+
PYTHON_VERSION: "3.13"
67+
LOKY_TEST_NO_LSCPU: "true"
68+
- name: linux-py39-joblib-tests
69+
os: ubuntu-latest
70+
PYTHON_VERSION: "3.9"
71+
JOBLIB_TESTS: "true"
72+
- name: linux-python-py39-high-memory
73+
os: ubuntu-latest
74+
PYTHON_VERSION: "3.9"
75+
RUN_MEMORY: "true"
76+
- name: linux-py39
77+
os: ubuntu-latest
78+
PYTHON_VERSION: "3.9"
79+
# Do not install psutil on Python 3.9 to have some CI runs that
80+
# tests that loky has no hard dependency on psutil.
81+
NO_PSUTIL: "true"
82+
83+
# Test that the frozen executables are working
84+
- name: windows-py312-frozen
85+
os: windows-latest
86+
PYTHON_VERSION: "3.12"
87+
PYINSTALLER_TESTS: "true"
88+
- name: macos-py312-frozen
89+
os: macos-latest
90+
PYTHON_VERSION: "3.12"
91+
PYINSTALLER_TESTS: "true"
92+
- name: linux-py312-frozen
93+
os: ubuntu-latest
94+
PYTHON_VERSION: "3.12"
95+
PYINSTALLER_TESTS: "true"
96+
97+
env: ${{ matrix }}
98+
99+
runs-on: ${{ matrix.os }}
100+
101+
defaults:
102+
run:
103+
# Need to use this shell to get conda working properly.
104+
# See https://github.com/marketplace/actions/setup-miniconda#important
105+
shell: ${{ matrix.os == 'windows-latest' && 'cmd /C CALL {0}' || 'bash -el {0}' }}
106+
107+
108+
steps:
109+
- name: Checkout code
110+
uses: actions/checkout@v3
111+
112+
- name: Setup conda
113+
uses: conda-incubator/setup-miniconda@v3
114+
with:
115+
auto-activate-base: true
116+
auto-update-conda: true
117+
miniforge-version: latest
118+
119+
- name: Install dependencies
120+
run: |
121+
bash -el continuous_integration/install.sh
122+
123+
- name: Run tests
124+
run: |
125+
bash -el continuous_integration/runtests.sh
126+
127+
- name: Upload to Codecov
128+
# always upload coverage even if tests fail
129+
if: ${{ matrix.JOBLIB_TESTS != 'true' && (success() || failure()) }}
130+
uses: codecov/codecov-action@v5
131+
with:
132+
files: coverage.xml

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
# General ignore rules
22
.*
3+
!.github
34
*sublime*
45
tags
6+
!.github
57

68
# Python running files
79
__pycache__

‎.pre-commit-config.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ repos:
66
- id: end-of-file-fixer
77
- id: trailing-whitespace
88
- repo: https://github.com/psf/black
9-
rev: 22.3.0
9+
rev: 25.1.0
1010
hooks:
1111
- id: black

‎.readthedocs.yml

+20-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,22 @@
1-
requirements_file: docs/requirements.txt
1+
# Read the Docs configuration file for Sphinx projects
2+
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
3+
version: 2
24

5+
# Set the OS, Python version and other tools you might need
6+
build:
7+
os: ubuntu-22.04
8+
tools:
9+
python: "3.11"
10+
11+
# Build documentation in the "docs/" directory with Sphinx
12+
sphinx:
13+
configuration: docs/conf.py
14+
15+
# Optional but recommended, declare the Python requirements required
16+
# to build your documentation
17+
# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
318
python:
4-
version: 3.8
19+
install:
20+
- method: pip
21+
path: .
22+
- requirements: docs/requirements.txt

‎CHANGES.md

+40-3
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,52 @@
1-
### 3.4.0 - in development
1+
### 3.5.0 - in development
2+
3+
- Avoid raising `DeprecationWarning` related to `os.fork` when running in a
4+
natively multi-threaded process. (#429).
5+
6+
- Fix a crash when calling commands that access `stdin` via `subprocess.run` in
7+
worker processes on POSIX systems. (#429).
8+
9+
- Automatically call `faulthandler.enable()` when starting loky worker
10+
processes to report more informative information (post-mortem Python
11+
tracebacks in particular) on worker crashs. (#419).
12+
13+
- Fix a random deadlock caused by a race condition at executor shutdown that
14+
was observed on Linux and Windows. (#438)
15+
16+
- Fix detection of the number of physical cores in
17+
`cpu_count(only_physical_cores=True)` on some Linux systems and recent
18+
Windows versions. (#425)
19+
20+
- Drop support for Python 3.7 and Python 3.8. (#409)
21+
22+
- Drop support for PyPy. (#427)
23+
24+
### 3.4.1 - 2023-06-29
25+
26+
- Fix compatibility with python3.7, which does not define
27+
a `_MAX_WINDOWS_WORKERS` constant. (#408)
28+
29+
### 3.4.0 - 2023-04-14
230

331
- Fix exception `__cause__` not being propagated with
4-
`tblib.pickling_support.install()` (#255).
32+
`tblib.pickling_support.install()`. (#255).
533

634
- Fix handling of CPU affinity by using `psutil`'s `cpu_affinity` on platforms
7-
that do not implement `os.sched_getaffinity`, such as PyPy (#381).
35+
that do not implement `os.sched_getaffinity`, such as PyPy. (#381).
36+
37+
- Make the executor's gc process more thread-safe, in particular for PyPy,
38+
where the gc calls can be run in any thread. (#384).
839

940
- Fix crash when using `max_workers > 61` on Windows. Loky will no longer
1041
attempt to use more than 61 workers on that platform (or 60 depending on the
1142
Python version). (#390).
1243

44+
- Fix loky compat with python 3.11 for nested calls. (#394).
45+
46+
- Adapt the cooldown strategy when shutingdown an executor with full
47+
`call_queue`. This should accelerate the time taken to shutdown
48+
in general, in particular on overloaded machines. (#399).
49+
1350
### 3.3.0 - 2022-09-15
1451

1552
- Fix worker management logic in `get_reusable_executor` to ensure

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ alt="Loky logo" width=96/></a>
55

66

77
# Reusable Process Pool Executor
8-
[![Build Status](https://dev.azure.com/joblib/loky/_apis/build/status/joblib.loky?branchName=master)](https://dev.azure.com/joblib/loky/_build/latest?definitionId=2&branchName=master)
8+
[![Build Status](https://github.com/joblib/loky/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/joblib/loky/actions/workflows/test.yml?query=branch%3Amaster)
99
[![Documentation Status](https://readthedocs.org/projects/loky/badge/?version=latest)](https://loky.readthedocs.io/en/latest/?badge=latest)
1010
[![codecov](https://codecov.io/gh/joblib/loky/branch/master/graph/badge.svg)](https://codecov.io/gh/joblib/loky)
1111
[![DOI](https://zenodo.org/badge/48578152.svg)](https://zenodo.org/badge/latestdoi/48578152)

‎continuous_integration/install.sh

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
# License: 3-clause BSD
4+
5+
set -xe
6+
7+
# Create new conda env
8+
conda config --set solver libmamba
9+
to_install="python=$PYTHON_VERSION pip numpy tblib viztracer"
10+
conda create -n testenv --yes -c conda-forge $to_install
11+
conda activate testenv
12+
13+
if [[ -z "$JOBLIB_TESTS" ]]; then
14+
# Install pytest timeout to fasten failure in deadlocking tests
15+
PIP_INSTALL_PACKAGES="pytest pytest-timeout coverage pytest-cov"
16+
fi
17+
18+
if [[ -z "$NO_PSUTIL" ]]; then
19+
PIP_INSTALL_PACKAGES="$PIP_INSTALL_PACKAGES psutil"
20+
fi
21+
22+
pip install $PIP_INSTALL_PACKAGES
23+
24+
pip install -v .

‎continuous_integration/install_coverage_subprocess_pth.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@
33
# http://coverage.readthedocs.io/en/latest/subprocess.html
44

55
import os.path as op
6-
from distutils.sysconfig import get_python_lib
6+
from sysconfig import get_path
77

88
FILE_CONTENT = """\
99
import coverage; coverage.process_startup()
1010
"""
1111

12-
filename = op.join(get_python_lib(), "coverage_subprocess.pth")
12+
# get_path("platlib") returns the location of the `site-packages` folder:
13+
# https://docs.python.org/3/library/sysconfig.html
14+
filename = op.join(get_path("platlib"), "coverage_subprocess.pth")
1315
with open(filename, mode="w") as f:
1416
f.write(FILE_CONTENT)
1517

‎continuous_integration/runtests.sh

+29-15
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,32 @@
1-
#!/usr/bin/env sh
1+
#!/bin/bash
22

33
# License: 3-clause BSD
44

5-
set -e
5+
set -xe
6+
7+
conda activate testenv
8+
69
which python
7-
python --version
8-
echo ${TOX_ENV}
10+
python -V
11+
python -c "import struct; print('platform: %d' % (8 * struct.calcsize('P')))"
12+
python -c "import loky; print('loky.cpu_count():', loky.cpu_count())"
13+
python -c "import os; print('os.cpu_count():', os.cpu_count())"
914

10-
if [ "$JOBLIB_TESTS" = "true" ]; then
15+
16+
if [[ "$JOBLIB_TESTS" == "true" ]]; then
1117
# Install joblib from pip, patch it to use this version of loky
1218
# and run the joblib tests with pytest.
13-
python -m venv venv/
14-
source ./venv/bin/activate
15-
which python
19+
LOKY_PATH=$(pwd)
20+
1621
git clone https://github.com/joblib/joblib.git src_joblib
1722
cd src_joblib
1823
pip install "pytest<7.0" # Need to update remove occurrences of pytest.warns(None)
1924
pip install threadpoolctl # required by some joblib tests
2025

2126
pip install -e .
2227
export JOBLIB=`python -c "import joblib; print(joblib.__path__[0])"`
23-
cp "$BUILD_SOURCESDIRECTORY"/continuous_integration/copy_loky.sh $JOBLIB/externals
24-
(cd $JOBLIB/externals && bash copy_loky.sh "$BUILD_SOURCESDIRECTORY")
28+
cp "$LOKY_PATH"/continuous_integration/copy_loky.sh $JOBLIB/externals
29+
(cd $JOBLIB/externals && bash copy_loky.sh "$LOKY_PATH")
2530
pytest -vl --ignore $JOBLIB/externals --pyargs joblib
2631
elif [ "$PYINSTALLER_TESTS" = "true" ]; then
2732
python -m venv venv/
@@ -44,14 +49,23 @@ elif [ "$PYINSTALLER_TESTS" = "true" ]; then
4449
else
4550
# Make sure that we have the python docker image cached locally to avoid
4651
# a timeout in a test that needs it.
47-
if [ "$(which docker)" != "" ] && [ "$(uname)" = "Linux" ]; then
52+
if [[ "$(which docker)" != "" ]] && [[ "$(uname)" = "Linux" ]]; then
4853
docker pull python:3.10
4954
fi
5055

51-
# Run the tests and collect trace coverage data both in the subprocesses
52-
# and its subprocesses.
53-
if [ "$RUN_MEMORY" != "true" ]; then
56+
# Enable coverage reporting from subprocesses.
57+
python continuous_integration/install_coverage_subprocess_pth.py
58+
59+
PYTEST_ARGS="-vl --timeout=120 --maxfail=5 --cov=loky --cov-report xml"
60+
61+
if [[ "$RUN_MEMORY" != "true" ]]; then
5462
PYTEST_ARGS="$PYTEST_ARGS --skip-high-memory"
5563
fi
56-
tox -v -e "${TOX_ENV}" -- ${PYTEST_ARGS} --junitxml="${JUNITXML}"
64+
65+
LOKY_MAX_DEPTH=3
66+
OMP_NUM_THREADS=4
67+
68+
# Run the tests and collect trace coverage data both in the subprocesses
69+
# and its subprocesses.
70+
pytest $PYTEST_ARGS .
5771
fi

‎docs/API.rst

+4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ API Reference
55
.. automodule:: loky
66
:members: get_reusable_executor
77

8+
.. autoclass:: ProcessPoolExecutor
9+
:members:
10+
:class-doc-from: init
11+
812

913
Task & results serialization
1014
----------------------------

‎examples/deadlocks/deadlock_on_failed_pickle.py

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from concurrent.futures is not robust to pickling error (at least in versions
77
3.6 and lower).
88
"""
9+
910
import argparse
1011

1112

‎examples/deadlocks/deadlock_on_shutdown.py

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from concurrent.futures is not robust to pickling error (at least in versions
77
3.6 and lower).
88
"""
9+
910
import argparse
1011

1112

‎examples/deadlocks/deadlock_results.py

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from concurrent.futures is not robust to pickling error (at least in versions
66
3.6 and lower).
77
"""
8+
89
import argparse
910
from pickle import UnpicklingError
1011

‎examples/deadlocks/deadlock_timeout_shutdown.py

-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
3.6 and lower).
77
"""
88

9-
109
import time
1110
from loky import ProcessPoolExecutor
1211
from loky.backend import get_context

0 commit comments

Comments
 (0)