Skip to content

Commit

Permalink
Remove support for dill as a serialization backend.
Browse files Browse the repository at this point in the history
  • Loading branch information
bluescarni committed Feb 20, 2024
1 parent 8f582ff commit 76d0e4a
Show file tree
Hide file tree
Showing 14 changed files with 22 additions and 126 deletions.
2 changes: 0 additions & 2 deletions doc/install.rst
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ pygmo has the following **mandatory** runtime dependencies:
Additionally, pygmo has the following **optional** runtime
dependencies:

* `dill <https://dill.readthedocs.io>`__, which can be used as an
alternative serialization backend,
* `Matplotlib <https://matplotlib.org/>`__, which is used by a few
plotting utilities,
* `NetworkX <https://networkx.github.io/>`__, which is used for
Expand Down
17 changes: 3 additions & 14 deletions pygmo/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -746,8 +746,7 @@ def set_serialization_backend(name):
The valid backends are:
* ``'pickle'`` (i.e., the standard Python :mod:`pickle` module),
* ``'cloudpickle'``,
* ``'dill'`` (from the `dill <https://pypi.org/project/dill/>`__ library).
* ``'cloudpickle'``.
.. warning::
Expand All @@ -760,8 +759,7 @@ def set_serialization_backend(name):
Raises:
TypeError: if *name* is not a :class:`str`
ValueError: if *name* is not one of ``['pickle', 'cloudpickle', 'dill']``
ImportError: if *name* is ``'dill'`` but the dill module is not installed
ValueError: if *name* is not one of ``['pickle', 'cloudpickle']``
"""
if not isinstance(name, str):
Expand All @@ -777,18 +775,9 @@ def set_serialization_backend(name):
_serialization_backend = pickle
elif name == "cloudpickle":
_serialization_backend = _cloudpickle
elif name == "dill":
try:
import dill

_serialization_backend = dill
except ImportError:
raise ImportError(
"The 'dill' serialization backend was specified, but the dill module is not installed."
)
else:
raise ValueError(
"The serialization backend '{}' is not valid. The valid backends are: ['pickle', 'cloudpickle', 'dill']".format(
"The serialization backend '{}' is not valid. The valid backends are: ['pickle', 'cloudpickle']".format(
name
)
)
Expand Down
11 changes: 1 addition & 10 deletions pygmo/_island_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -501,17 +501,8 @@ def run_basic_tests(self):
from . import mp_island
from copy import copy, deepcopy

has_dill = False
try:
import dill
from pickle import dumps, loads

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
# Try shutting down a few times, to confirm that the second
# and third shutdowns don't do anything.
mp_island.shutdown_pool()
Expand Down
60 changes: 7 additions & 53 deletions pygmo/_py_islands.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,8 @@
def _evolve_func_mp_pool(ser_algo_pop):
# The evolve function that is actually run from the separate processes
# in mp_island (when using the pool).
has_dill = False
try:
import dill

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
from pickle import dumps, loads

algo, pop = loads(ser_algo_pop)
new_pop = algo.evolve(pop)
return dumps((algo, new_pop))
Expand All @@ -44,17 +35,8 @@ def _evolve_func_mp_pipe(conn, ser_algo_pop):
# of a child process happens in a separate thread and Python disallows messing
# with signal handlers from a thread different from the main one :(
with _temp_disable_sigint():
has_dill = False
try:
import dill
from pickle import dumps, loads

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
try:
algo, pop = loads(ser_algo_pop)
new_pop = algo.evolve(pop)
Expand Down Expand Up @@ -227,17 +209,8 @@ def run_evolve(self, algo, pop):
# that if there are serialization errors, we catch them early here rather
# than failing in the bootstrap phase of the remote process, which
# can lead to hangups.
has_dill = False
try:
import dill
from pickle import dumps, loads

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
ser_algo_pop = dumps((algo, pop))

if self._use_pool:
Expand Down Expand Up @@ -470,17 +443,8 @@ def shutdown_pool():
def _evolve_func_ipy(ser_algo_pop):
# The evolve function that is actually run from the separate processes
# in ipyparallel_island.
has_dill = False
try:
import dill

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
from pickle import dumps, loads

algo, pop = loads(ser_algo_pop)
new_pop = algo.evolve(pop)
return dumps((algo, new_pop))
Expand Down Expand Up @@ -613,17 +577,7 @@ def run_evolve(self, algo, pop):
# serialization errors early.
from ._ipyparallel_utils import _make_ipyparallel_view

has_dill = False
try:
import dill

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
from pickle import dumps, loads

ser_algo_pop = dumps((algo, pop))
with ipyparallel_island._view_lock:
Expand Down
40 changes: 2 additions & 38 deletions pygmo/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,6 @@ def run_s11n_test(self):
from . import set_serialization_backend as ssb, get_serialization_backend as gsb
from . import problem, island, de

has_dill = False
try:
import dill

has_dill = True
except ImportError:
pass

# Default s11n backend.
self.assertTrue(gsb() == clpickle)

Expand All @@ -93,19 +85,10 @@ def run_s11n_test(self):
ssb("hello")
err = cm.exception
self.assertEqual(
"The serialization backend 'hello' is not valid. The valid backends are: ['pickle', 'cloudpickle', 'dill']",
"The serialization backend 'hello' is not valid. The valid backends are: ['pickle', 'cloudpickle']",
str(err),
)

if not has_dill:
with self.assertRaises(ImportError) as cm:
ssb("dill")
err = cm.exception
self.assertEqual(
"The 'dill' serialization backend was specified, but the dill module is not installed.",
str(err),
)

ssb("pickle")
self.assertTrue(gsb() == pickle)

Expand All @@ -115,16 +98,6 @@ def run_s11n_test(self):
isl = island(prob=p, algo=de(gen=500), size=20)
self.assertEqual(str(pickle.loads(pickle.dumps(isl))), str(isl))

# Try with dill as well, if available.
if has_dill:
ssb("dill")
self.assertTrue(gsb() == dill)

p = problem(_prob())
self.assertEqual(str(dill.loads(dill.dumps(p))), str(p))
isl = island(prob=p, algo=de(gen=500), size=20)
self.assertEqual(str(dill.loads(dill.dumps(isl))), str(isl))

# Reset to cloudpickle before exiting.
ssb("cloudpickle")
self.assertTrue(gsb() == clpickle)
Expand Down Expand Up @@ -1495,17 +1468,8 @@ def run_pickle_tests(self):
migrant_handling,
)

has_dill = False
try:
import dill
from pickle import dumps, loads

has_dill = True
except ImportError:
pass
if has_dill:
from dill import dumps, loads
else:
from pickle import dumps, loads
a = archipelago(5, algo=de(), prob=rosenbrock(), pop_size=10)
self.assertEqual(repr(a), repr(loads(dumps(a))))
a = archipelago(5, algo=de(), prob=_prob(), pop_size=10, udi=mp_island())
Expand Down
2 changes: 1 addition & 1 deletion tools/circleci_bionic_conda_pagmo_head_310.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 sphinx=4.5.0 myst-nb sphinx-book-theme scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx numba pybind11 sphinx=4.5.0 myst-nb sphinx-book-theme scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/circleci_bionic_conda_pagmo_head_38.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx numba pybind11 scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/circleci_bionic_conda_pagmo_head_39.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.9 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.9 numpy cloudpickle networkx numba pybind11 scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/circleci_ubuntu_arm64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge
export deps_dir=$HOME/local
export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx numba pybind11 scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/gha_deploydocs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ bash miniconda.sh -b -p $HOME/miniconda
conda config --add channels conda-forge
conda config --set channel_priority strict
conda install mamba
mamba create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 sphinx=4.5.0 myst-nb sphinx-book-theme scipy
mamba create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx numba pybind11 sphinx=4.5.0 myst-nb sphinx-book-theme scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/gha_macos-10.15-py310.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda config --add channels conda-forge
conda config --set channel_priority strict
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 sphinx myst-nb sphinx-book-theme scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt ipopt boost-cpp tbb tbb-devel python=3.10 numpy cloudpickle networkx numba pybind11 sphinx myst-nb sphinx-book-theme scipy
source activate $deps_dir

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/gha_manylinux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ fi
# Python mandatory deps.
/opt/python/${PYTHON_DIR}/bin/pip install cloudpickle numpy
# Python optional deps.
/opt/python/${PYTHON_DIR}/bin/pip install dill==0.3.5.1 networkx ipyparallel scipy
/opt/python/${PYTHON_DIR}/bin/pip install networkx ipyparallel scipy

# In the pagmo2/manylinux228_x86_64_with_deps:latest image in dockerhub
# the working directory is /root/install, we will install pagmo there
Expand Down
2 changes: 1 addition & 1 deletion tools/gha_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Powershell script
# Install conda environment
conda config --set always_yes yes
conda create --name pygmo cmake eigen nlopt ipopt boost-cpp tbb tbb-devel numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 scipy
conda create --name pygmo cmake eigen nlopt ipopt boost-cpp tbb tbb-devel numpy cloudpickle networkx numba pybind11 scipy
conda activate pygmo

# Install pagmo.
Expand Down
2 changes: 1 addition & 1 deletion tools/travis_ubuntu_ppc64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge
export deps_dir=$HOME/local
export PATH="$HOME/miniconda/bin:$PATH"
bash miniconda.sh -b -p $HOME/miniconda
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx dill=0.3.5.1 numba pybind11 scipy
conda create -y -q -p $deps_dir c-compiler cxx-compiler cmake eigen nlopt boost-cpp tbb tbb-devel python=3.8 numpy cloudpickle networkx numba pybind11 scipy
source activate $deps_dir

# Install pagmo.
Expand Down

0 comments on commit 76d0e4a

Please sign in to comment.