Merge branch 'master' into more-linalg2

Author: asmeurer

.github/workflows/numpy.yml

@@ -8,7 +8,7 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- python-version: [3.8, 3.9]
+ python-version: ["3.10", "3.11"]
 
  steps:
  - name: Checkout array-api-tests
@@ -22,60 +22,10 @@ jobs:
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip
- python -m pip install numpy==1.22.1
+ python -m pip install numpy==1.26.2
  python -m pip install -r requirements.txt
  - name: Run the test suite
  env:
  ARRAY_API_TESTS_MODULE: numpy.array_api
  run: |
- # Skip testing functions with known issues
- cat << EOF >> skips.txt
-
- # copy not implemented
- array_api_tests/test_creation_functions.py::test_asarray_arrays
- # https://github.com/numpy/numpy/issues/20870
- array_api_tests/test_data_type_functions.py::test_can_cast
- # The return dtype for trace is not consistent in the spec
- # https://github.com/data-apis/array-api/issues/202#issuecomment-952529197
- array_api_tests/test_linalg.py::test_trace
- # Various fixes to vector_norm are in
- # https://github.com/numpy/numpy/pull/21084.
- array_api_tests/test_linalg.py::test_vector_norm
- # waiting on NumPy to allow/revert distinct NaNs for np.unique
- # https://github.com/numpy/numpy/issues/20326#issuecomment-1012380448
- array_api_tests/test_set_functions.py
-
- # https://github.com/numpy/numpy/issues/21373
- array_api_tests/test_array_object.py::test_getitem
-
- # missing copy arg
- array_api_tests/test_signatures.py::test_func_signature[reshape]
-
- # https://github.com/numpy/numpy/issues/21211
- array_api_tests/test_special_cases.py::test_iop[__iadd__(x1_i is -0 and x2_i is -0) -> -0]
- # https://github.com/numpy/numpy/issues/21213
- array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -infinity and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +infinity]
- array_api_tests/test_special_cases.py::test_iop[__ipow__(x1_i is -0 and x2_i > 0 and not (x2_i.is_integer() and x2_i % 2 == 1)) -> +0]
- # noted diversions from spec
- array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_binary[floor_divide(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
- array_api_tests/test_special_cases.py::test_binary[floor_divide(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
- array_api_tests/test_special_cases.py::test_binary[__floordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity and isfinite(x2_i) and x2_i < 0) -> +infinity]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
- array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
-
- EOF
-
- pytest -v -rxXfE --ci
+ pytest -v -rxXfE --ci --skips-file numpy-skips.txt

.gitignore

@@ -117,6 +117,10 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+# IDE
+.idea/
+.vscode/
+
 # mkdocs documentation
 /site
 

README.md

@@ -36,13 +36,7 @@ You need to specify the array library to test. It can be specified via the
 $ export ARRAY_API_TESTS_MODULE=numpy.array_api
 ```
 
-Alternately, change the `array_module` variable in `array_api_tests/_array_module.py`
-line, e.g.
-
-```diff
-- array_module = None
-+ import numpy.array_api as array_module
-```
+Alternately, import/define the `xp` variable in `array_api_tests/__init__.py`.
 
 ### Run the suite
 
@@ -160,6 +154,13 @@ library to fail.
 
 ### Configuration
 
+#### API version
+
+You can specify the API version to use when testing via the
+`ARRAY_API_TESTS_VERSION` environment variable. Currently this defaults to the
+array module's `__array_api_version__` value, and if that attribute doesn't
+exist then we fallback to `"2021.12"`.
+
 #### CI flag
 
 Use the `--ci` flag to run only the primary and special cases tests. You can
@@ -178,13 +179,22 @@ By default, tests for the optional Array API extensions such as
 will be skipped if not present in the specified array module. You can purposely
 skip testing extension(s) via the `--disable-extension` option.
 
-#### Skip test cases
+#### Skip or XFAIL test cases
+
+Test cases you want to skip can be specified in a skips or XFAILS file. The
+difference between skip and XFAIL is that XFAIL tests are still run and
+reported as XPASS if they pass.
+
+By default, the skips and xfails files are `skips.txt` and `fails.txt` in the root
+of this repository, but any file can be specified with the `--skips-file` and
+`--xfails-file` command line flags.
 
-Test cases you want to skip can be specified in a `skips.txt` file in the root
-of this repository, e.g.:
+The files should list the test ids to be skipped/xfailed. Empty lines and
+lines starting with `#` are ignored. The test id can be any substring of the
+test ids to skip/xfail.
 
 ```
-# ./skips.txt
+# skips.txt or xfails.txt
 # Line comments can be denoted with the hash symbol (#)
 
 # Skip specific test case, e.g. when argsort() does not respect relative order
@@ -200,39 +210,81 @@ array_api_tests/test_add[__iadd__(x, s)]
 array_api_tests/test_set_functions.py
 ```
 
-For GitHub Actions, you might like to keep everything in the workflow config
-instead of having a seperate `skips.txt` file, e.g.:
+Here is an example GitHub Actions workflow file, where the xfails are stored
+in `array-api-tests.xfails.txt` in the base of the `your-array-library` repo.
+
+If you want, you can use `-o xfail_strict=True`, which causes XPASS tests (XFAIL
+tests that actually pass) to fail the test suite. However, be aware that
+XFAILures can be flaky (see below, so this may not be a good idea unless you
+use some other mitigation of such flakyness).
+
+If you don't want this behavior, you can remove it, or use `--skips-file`
+instead of `--xfails-file`.
 
 ```yaml
 # ./.github/workflows/array_api.yml
-...
- ...
- - name: Run the test suite
+jobs:
+ tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ['3.8', '3.9', '3.10', '3.11']
+
+ steps:
+ - name: Checkout <your array library>
+ uses: actions/checkout@v3
+ with:
+ path: your-array-library
+
+ - name: Checkout array-api-tests
+ uses: actions/checkout@v3
+ with:
+ repository: data-apis/array-api-tests
+ submodules: 'true'
+ path: array-api-tests
+
+ - name: Run the array API test suite
  env:
  ARRAY_API_TESTS_MODULE: your.array.api.namespace
  run: |
- # Skip test cases with known issues
- cat << EOF >> skips.txt
-
- # Comments can still work here
- array_api_tests/test_sorting_functions.py::test_argsort
- array_api_tests/test_add[__iadd__(x1, x2)]
- array_api_tests/test_add[__iadd__(x, s)]
- array_api_tests/test_set_functions.py
-
- EOF
-
- pytest -v -rxXfE --ci
+ export PYTHONPATH="${GITHUB_WORKSPACE}/your-array-library"
+ cd ${GITHUB_WORKSPACE}/array-api-tests
+ pytest -v -rxXfE --ci --xfails-file ${GITHUB_WORKSPACE}/your-array-library/array-api-tests-xfails.txt array_api_tests/
 ```
 
+> **Warning**
+>
+> XFAIL tests that use Hypothesis (basically every test in the test suite except
+> those in test_has_names.py) can be flaky, due to the fact that Hypothesis
+> might not always run the test with an input that causes the test to fail.
+> There are several ways to avoid this problem:
+>
+> - Increase the maximum number of examples, e.g., by adding `--max-examples
+> 200` to the test command (the default is `100`, see below). This will
+> make it more likely that the failing case will be found, but it will also
+> make the tests take longer to run.
+> - Don't use `-o xfail_strict=True`. This will make it so that if an XFAIL
+> test passes, it will alert you in the test summary but will not cause the
+> test run to register as failed.
+> - Use skips instead of XFAILS. The difference between XFAIL and skip is that
+> a skipped test is never run at all, whereas an XFAIL test is always run
+> but ignored if it fails.
+> - Save the [Hypothesis examples
+> database](https://hypothesis.readthedocs.io/en/latest/database.html)
+> persistently on CI. That way as soon as a run finds one failing example,
+> it will always re-run future runs with that example. But note that the
+> Hypothesis examples database may be cleared when a new version of
+> Hypothesis or the test suite is released.
+
 #### Max examples
 
 The tests make heavy use
 [Hypothesis](https://hypothesis.readthedocs.io/en/latest/). You can configure
-how many examples are generated using the `--max-examples` flag, which defaults
-to 100. Lower values can be useful for quick checks, and larger values should
-result in more rigorous runs. For example, `--max-examples 10_000` may find bugs
-where default runs don't but will take much longer to run.
+how many examples are generated using the `--max-examples` flag, which
+defaults to `100`. Lower values can be useful for quick checks, and larger
+values should result in more rigorous runs. For example, `--max-examples
+10_000` may find bugs where default runs don't but will take much longer to
+run.
 
 
 ## Contributing

array-api

@@ -1,11 +1,36 @@
+import os
 from functools import wraps
+from importlib import import_module
 
 from hypothesis import strategies as st
 from hypothesis.extra import array_api
 
-from ._array_module import mod as _xp
+from . import _version
 
-__all__ = ["xps"]
+__all__ = ["xp", "api_version", "xps"]
+
+
+# You can comment the following out and instead import the specific array module
+# you want to test, e.g. `import numpy.array_api as xp`.
+if "ARRAY_API_TESTS_MODULE" in os.environ:
+ xp_name = os.environ["ARRAY_API_TESTS_MODULE"]
+ _module, _sub = xp_name, None
+ if "." in xp_name:
+ _module, _sub = xp_name.split(".", 1)
+ xp = import_module(_module)
+ if _sub:
+ try:
+ xp = getattr(xp, _sub)
+ except AttributeError:
+ # _sub may be a submodule that needs to be imported. WE can't
+ # do this in every case because some array modules are not
+ # submodules that can be imported (like mxnet.nd).
+ xp = import_module(xp_name)
+else:
+ raise RuntimeError(
+ "No array module specified - either edit __init__.py or set the "
+ "ARRAY_API_TESTS_MODULE environment variable."
+ )
 
 
 # We monkey patch floats() to always disable subnormals as they are out-of-scope
@@ -41,9 +66,9 @@ def _from_dtype(*a, **kw):
  pass
 
 
-xps = array_api.make_strategies_namespace(_xp, api_version="2021.12")
-
-
-from . import _version
+api_version = os.getenv(
+ "ARRAY_API_TESTS_VERSION", getattr(xp, "__array_api_version__", "2021.12")
+)
+xps = array_api.make_strategies_namespace(xp, api_version=api_version)
 
 __version__ = _version.get_versions()["version"]

array_api_tests/__init__.py

@@ -1,35 +1,5 @@
-import os
-from importlib import import_module
+from . import stubs, xp
 
-from . import stubs
-
-# Replace this with a specific array module to test it, for example,
-#
-# import numpy as array_module
-array_module = None
-
-if array_module is None:
- if 'ARRAY_API_TESTS_MODULE' in os.environ:
- mod_name = os.environ['ARRAY_API_TESTS_MODULE']
- _module, _sub = mod_name, None
- if '.' in mod_name:
- _module, _sub = mod_name.split('.', 1)
- mod = import_module(_module)
- if _sub:
- try:
- mod = getattr(mod, _sub)
- except AttributeError:
- # _sub may be a submodule that needs to be imported. WE can't
- # do this in every case because some array modules are not
- # submodules that can be imported (like mxnet.nd).
- mod = import_module(mod_name)
- else:
- raise RuntimeError("No array module specified. Either edit _array_module.py or set the ARRAY_API_TESTS_MODULE environment variable")
-else:
- mod = array_module
- mod_name = mod.__name__
-# Names from the spec. This is what should actually be imported from this
-# file.
 
 class _UndefinedStub:
  """
@@ -45,7 +15,7 @@ def __init__(self, name):
  self.name = name
 
  def _raise(self, *args, **kwargs):
- raise AssertionError(f"{self.name} is not defined in {mod_name}")
+ raise AssertionError(f"{self.name} is not defined in {xp.__name__}")
 
  def __repr__(self):
  return f"<undefined stub for {self.name!r}>"
@@ -58,13 +28,15 @@ def __repr__(self):
  "uint8", "uint16", "uint32", "uint64",
  "int8", "int16", "int32", "int64",
  "float32", "float64",
+ "complex64", "complex128",
 ]
 _constants = ["e", "inf", "nan", "pi"]
 _funcs = [f.__name__ for funcs in stubs.category_to_funcs.values() for f in funcs]
-_top_level_attrs = _dtypes + _constants + _funcs + stubs.EXTENSIONS
+_funcs += ["take", "isdtype", "conj", "imag", "real"] # TODO: bump spec and update array-api-tests to new spec layout
+_top_level_attrs = _dtypes + _constants + _funcs + stubs.EXTENSIONS + ["fft"]
 
 for attr in _top_level_attrs:
  try:
- globals()[attr] = getattr(mod, attr)
+ globals()[attr] = getattr(xp, attr)
  except AttributeError:
  globals()[attr] = _UndefinedStub(attr)