[MRG] PyPy support for all but a couple of estimators (scikit-learn#11010)

Author: rlamy

.circleci/config.yml

@@ -65,6 +65,22 @@ jobs:
  path: ~/log.txt
  destination: log.txt
 
+ pypy3:
+ docker:
+ - image: pypy:3-6.0.0
+ steps:
+ - restore_cache:
+ keys:
+ - pypy3-ccache-{{ .Branch }}
+ - pypy3-ccache
+ - checkout
+ - run: ./build_tools/circle/build_test_pypy.sh
+ - save_cache:
+ key: pypy3-ccache-{{ .Branch }}-{{ .BuildNum }}
+ paths:
+ - ~/.ccache
+ - ~/.cache/pip
+
  deploy:
  docker:
  - image: circleci/python:3.6.1
@@ -88,6 +104,7 @@ workflows:
  jobs:
  - python3
  - python2
+ - pypy3
  - deploy:
  requires:
  - python3

build_tools/circle/build_test_pypy.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -x
+set -e
+
+apt-get -yq update
+apt-get -yq install libatlas-dev libatlas-base-dev liblapack-dev gfortran ccache
+
+pip install virtualenv
+
+if command -v pypy3; then
+ virtualenv -p $(command -v pypy3) pypy-env
+elif command -v pypy; then
+ virtualenv -p $(command -v pypy) pypy-env
+fi
+
+source pypy-env/bin/activate
+
+python --version
+which python
+
+pip install --extra-index https://antocuni.github.io/pypy-wheels/ubuntu numpy==1.14.4 Cython pytest
+pip install "scipy>=1.1.0" sphinx numpydoc docutils
+
+ccache -M 512M
+export CCACHE_COMPRESS=1
+export PATH=/usr/lib/ccache:$PATH
+
+pip install -e .
+
+make test

conftest.py

@@ -5,13 +5,23 @@
 # doc/modules/clustering.rst and use sklearn from the local folder rather than
 # the one from site-packages.
 
+import platform
 from distutils.version import LooseVersion
 
 import pytest
 from _pytest.doctest import DoctestItem
 
 
 def pytest_collection_modifyitems(config, items):
+
+ # FeatureHasher is not compatible with PyPy
+ if platform.python_implementation() == 'PyPy':
+ skip_marker = pytest.mark.skip(
+ reason='FeatureHasher is not compatible with PyPy')
+ for item in items:
+ if item.name == 'sklearn.feature_extraction.hashing.FeatureHasher':
+ item.add_marker(skip_marker)
+
  # numpy changed the str/repr formatting of numpy arrays in 1.14. We want to
  # run doctests only for numpy >= 1.14.
  skip_doctests = True

doc/conftest.py

@@ -1,9 +1,11 @@
+import os
 from os.path import exists
 from os.path import join
 import warnings
 
 import numpy as np
 
+from sklearn.utils import IS_PYPY
 from sklearn.utils.testing import SkipTest
 from sklearn.utils.testing import check_skip_network
 from sklearn.datasets import get_data_home
@@ -56,6 +58,8 @@ def setup_twenty_newsgroups():
 
 
 def setup_working_with_text_data():
+ if IS_PYPY and os.environ.get('CI', None):
+ raise SkipTest('Skipping too slow test with PyPy on CI')
  check_skip_network()
  cache_path = _pkl_filepath(get_data_home(), CACHE_NAME)
  if not exists(cache_path):
@@ -98,6 +102,8 @@ def pytest_runtest_setup(item):
  setup_working_with_text_data()
  elif fname.endswith('modules/compose.rst') or is_index:
  setup_compose()
+ elif IS_PYPY and fname.endswith('modules/feature_extraction.rst'):
+ raise SkipTest('FeatureHasher is not compatible with PyPy')
  elif fname.endswith('modules/impute.rst'):
  setup_impute()
  elif fname.endswith('statistical_inference/unsupervised_learning.rst'):

doc/developers/advanced_installation.rst

@@ -38,6 +38,12 @@ Scikit-learn requires:
 - NumPy (>= 1.8.2),
 - SciPy (>= 0.13.3).
 
+.. note::
+
+ For installing on PyPy, PyPy3-v5.10+, Numpy 1.14.0+, and scipy 1.1.0+
+ are required. For PyPy, only installation instructions with pip apply.
+
+
 Building Scikit-learn also requires
 
 - Cython >=0.23 

doc/developers/contributing.rst

@@ -352,7 +352,7 @@ and Cython optimizations.
 
  * Travis is used for testing on Linux platforms
  * Appveyor is used for testing on Windows platforms
- * CircleCI is used to build the docs for viewing
+ * CircleCI is used to build the docs for viewing and for testing with PyPy on Linux
 
  Please note that if one of the following markers appear in the latest commit
  message, the following actions are taken.

doc/faq.rst

@@ -179,12 +179,10 @@ careful choice of algorithms.
 Do you support PyPy?
 --------------------
 
-In case you didn't know, `PyPy <http://pypy.org/>`_ is the new, fast,
-just-in-time compiling Python implementation. We don't support it.
-When the `NumPy support <http://buildbot.pypy.org/numpy-status/latest.html>`_
-in PyPy is complete or near-complete, and SciPy is ported over as well,
-we can start thinking of a port.
-We use too much of NumPy to work with a partial implementation.
+In case you didn't know, `PyPy <http://pypy.org/>`_ is an alternative
+Python implementation with a built-in just-in-time compiler. Experimental
+support for PyPy3-v5.10+ has been added, which requires Numpy 1.14.0+,
+and scipy 1.1.0+.
 
 How do I deal with string data (or trees, graphs...)?
 -----------------------------------------------------

doc/install.rst

@@ -52,6 +52,12 @@ it as ``scikit-learn[alldeps]``. The most common use case for this is in a
 application or a Docker image. This option is not intended for manual
 installation from the command line.
 
+.. note::
+
+ For installing on PyPy, PyPy3-v5.10+, Numpy 1.14.0+, and scipy 1.1.0+
+ are required.
+
+
 For installation instructions for more distributions see
 :ref:`other distributions <install_by_distribution>`.
 For compiling the development version from source, or building the package

doc/whats_new/v0.20.rst

@@ -431,6 +431,12 @@ Miscellaneous
  :issue:`9101` by :user:`alex-33 <alex-33>`
  and :user:`Maskani Filali Mohamed <maskani-moh>`.
 
+- Add almost complete PyPy 3 support. Known unsupported functionalities are
+ :func:`datasets.load_svmlight_file`, :class:`feature_extraction.FeatureHasher` and
+ :class:`feature_extraction.text.HashingVectorizer`. For running on PyPy, PyPy3-v5.10+,
+ Numpy 1.14.0+, and scipy 1.1.0+ are required.
+ :issue:`11010` by :user:`Ronan Lamy <rlamy>` and `Roman Yurchak`_.
+
 Bug fixes
 .........
 

setup.py

@@ -3,10 +3,10 @@
 # Copyright (C) 2007-2009 Cournapeau David <cournape@gmail.com>
 # 2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
 # License: 3-clause BSD
-descr = """A set of python modules for machine learning and data mining"""
 
 import sys
 import os
+import platform
 import shutil
 from distutils.command.clean import clean as Clean
 from pkg_resources import parse_version
@@ -41,8 +41,12 @@
 
 VERSION = sklearn.__version__
 
-SCIPY_MIN_VERSION = '0.13.3'
-NUMPY_MIN_VERSION = '1.8.2'
+if platform.python_implementation() == 'PyPy':
+ SCIPY_MIN_VERSION = '1.1.0'
+ NUMPY_MIN_VERSION = '1.14.0'
+else:
+ SCIPY_MIN_VERSION = '0.13.3'
+ NUMPY_MIN_VERSION = '1.8.2'
 
 
 # Optional setuptools features
@@ -185,6 +189,10 @@ def setup_package():
  'Programming Language :: Python :: 3.4',
  'Programming Language :: Python :: 3.5',
  'Programming Language :: Python :: 3.6',
+ ('Programming Language :: Python :: '
+ 'Implementation :: CPython'),
+ ('Programming Language :: Python :: '
+ 'Implementation :: PyPy')
  ],
  cmdclass=cmdclass,
  install_requires=[