[MRG] set blockwise diagonals to zero for euclidean distance (scikit-learn#12612)

Author: amueller

doc/whats_new/v0.20.rst

@@ -120,6 +120,10 @@ Changelog
  set to "euclidean". :issue:`12481` by
  :user:`Jérémie du Boisberranger <jeremiedbb>`.
 
+- |Fix| Fixed a bug in :func:`metrics.pairwise.pairwise_distances_chunked`
+ which didn't ensure the diagonal is zero for euclidean distances.
+ :issue:`12612` by :user:`Andreas Müller <amueller>`.
+
 - |API| The :func:`metrics.calinski_harabaz_score` has been renamed to
  :func:`metrics.calinski_harabasz_score` and will be removed in version 0.23.
  :issue:`12211` by :user:`Lisa Thomas <LisaThomas9>`,

sklearn/metrics/pairwise.py

@@ -1271,6 +1271,12 @@ def pairwise_distances_chunked(X, Y=None, reduce_func=None,
  X_chunk = X[sl]
  D_chunk = pairwise_distances(X_chunk, Y, metric=metric,
  n_jobs=n_jobs, **kwds)
+ if ((X is Y or Y is None)
+ and PAIRWISE_DISTANCE_FUNCTIONS.get(metric, None)
+ is euclidean_distances):
+ # zeroing diagonal, taking care of aliases of "euclidean",
+ # i.e. "l2"
+ D_chunk.flat[sl.start::_num_samples(X) + 1] = 0
  if reduce_func is not None:
  chunk_size = D_chunk.shape[0]
  D_chunk = reduce_func(D_chunk, sl.start)

sklearn/metrics/tests/test_pairwise.py

@@ -473,7 +473,7 @@ def check_pairwise_distances_chunked(X, Y, working_memory, metric='euclidean'):
  metric=metric)
  assert isinstance(gen, GeneratorType)
  blockwise_distances = list(gen)
- Y = np.array(X if Y is None else Y)
+ Y = X if Y is None else Y
  min_block_mib = len(Y) * 8 * 2 ** -20
 
  for block in blockwise_distances:
@@ -485,6 +485,18 @@ def check_pairwise_distances_chunked(X, Y, working_memory, metric='euclidean'):
  assert_array_almost_equal(blockwise_distances, S)
 
 
+@pytest.mark.parametrize(
+ 'metric',
+ ('euclidean', 'l2', 'sqeuclidean'))
+def test_pairwise_distances_chunked_diagonal(metric):
+ rng = np.random.RandomState(0)
+ X = rng.normal(size=(1000, 10), scale=1e10)
+ chunks = list(pairwise_distances_chunked(X, working_memory=1,
+ metric=metric))
+ assert len(chunks) > 1
+ assert_array_almost_equal(np.diag(np.vstack(chunks)), 0, decimal=10)
+
+
 @ignore_warnings
 def test_pairwise_distances_chunked():
  # Test the pairwise_distance helper function.