New, restricted design that fixes circular dependency issues.

Author: fchollet

keras_preprocessing/__init__.py

@@ -4,27 +4,35 @@
 from __future__ import division
 from __future__ import print_function
 
-_KERAS_MODULE = None
+_KERAS_BACKEND = None
+_KERAS_UTILS = None
 
 
-def keras_module():
- global _KERAS_MODULE
- if _KERAS_MODULE is None:
- # Use `import keras` as default
- set_keras_module('keras')
- if _KERAS_MODULE == 'tensorflow.keras':
- # Due to TF namespace structure,
- # can't `__import__('tensorflow.keras')`.
- # Use workaround.
- tf = __import__('tensorflow')
- keras = tf.keras
- else:
- keras = __import__(_KERAS_MODULE, fromlist=['keras'])
- # TODO: check that the Keras version is compatible with
- # the current module.
- return keras
+def set_keras_submodules(backend, utils):
+ global _KERAS_BACKEND
+ global _KERAS_UTILS
+ _KERAS_BACKEND = backend
+ _KERAS_UTILS = utils
 
 
-def set_keras_module(module):
- global _KERAS_MODULE
- _KERAS_MODULE = module
+def get_keras_submodule(name):
+ if name not in {'backend', 'utils'}:
+ raise ImportError(
+ 'Can only retrieve "backend" and "utils". '
+ 'Requested: %s' % name)
+ if _KERAS_BACKEND is None or _KERAS_UTILS is None:
+ raise ImportError('You need to first `import keras` '
+ 'in order to use `keras_preprocessing`. '
+ 'For instance, you can do:\n\n'
+ '```\n'
+ 'import keras\n'
+ 'from keras_preprocessing import image\n'
+ '```\n\n'
+ 'Or, preferably, this equivalent formulation:\n\n'
+ '```\n'
+ 'from keras import preprocessing\n'
+ '```\n')
+ if name == 'backend':
+ return _KERAS_BACKEND
+ elif name == 'utils':
+ return _KERAS_UTILS

keras_preprocessing/image.py

@@ -15,9 +15,10 @@
 import multiprocessing.pool
 from functools import partial
 
-from . import keras_module
+from . import get_keras_submodule
 
-keras = keras_module()
+backend = get_keras_submodule('backend')
+keras_utils = get_keras_submodule('utils')
 
 try:
  from PIL import ImageEnhance
@@ -354,13 +355,13 @@ def array_to_img(x, data_format=None, scale=True):
  if pil_image is None:
  raise ImportError('Could not import PIL.Image. '
  'The use of `array_to_img` requires PIL.')
- x = np.asarray(x, dtype=keras.backend.floatx())
+ x = np.asarray(x, dtype=backend.floatx())
  if x.ndim != 3:
  raise ValueError('Expected image array to have rank 3 (single image). '
  'Got array with shape:', x.shape)
 
  if data_format is None:
- data_format = keras.backend.image_data_format()
+ data_format = backend.image_data_format()
  if data_format not in {'channels_first', 'channels_last'}:
  raise ValueError('Invalid data_format:', data_format)
 
@@ -400,13 +401,13 @@ def img_to_array(img, data_format=None):
  ValueError: if invalid `img` or `data_format` is passed.
  """
  if data_format is None:
- data_format = keras.backend.image_data_format()
+ data_format = backend.image_data_format()
  if data_format not in {'channels_first', 'channels_last'}:
  raise ValueError('Unknown data_format: ', data_format)
  # Numpy array x has format (height, width, channel)
  # or (channel, height, width)
  # but original PIL image has format (width, height, channel)
- x = np.asarray(img, dtype=keras.backend.floatx())
+ x = np.asarray(img, dtype=backend.floatx())
  if len(x.shape) == 3:
  if data_format == 'channels_first':
  x = x.transpose(2, 0, 1)
@@ -697,7 +698,7 @@ def __init__(self,
  data_format=None,
  validation_split=0.0):
  if data_format is None:
- data_format = keras.backend.image_data_format()
+ data_format = backend.image_data_format()
  self.featurewise_center = featurewise_center
  self.samplewise_center = samplewise_center
  self.featurewise_std_normalization = featurewise_std_normalization
@@ -949,7 +950,7 @@ def standardize(self, x):
  if self.samplewise_center:
  x -= np.mean(x, keepdims=True)
  if self.samplewise_std_normalization:
- x /= (np.std(x, keepdims=True) + keras.backend.epsilon())
+ x /= (np.std(x, keepdims=True) + backend.epsilon())
 
  if self.featurewise_center:
  if self.mean is not None:
@@ -961,7 +962,7 @@ def standardize(self, x):
  'first by calling `.fit(numpy_data)`.')
  if self.featurewise_std_normalization:
  if self.std is not None:
- x /= (self.std + keras.backend.epsilon())
+ x /= (self.std + backend.epsilon())
  else:
  warnings.warn('This ImageDataGenerator specifies '
  '`featurewise_std_normalization`, '
@@ -1165,7 +1166,7 @@ def fit(self, x,
  this is how many augmentation passes over the data to use.
  seed: Int (default: None). Random seed.
  """
- x = np.asarray(x, dtype=keras.backend.floatx())
+ x = np.asarray(x, dtype=backend.floatx())
  if x.ndim != 4:
  raise ValueError('Input to `.fit()` should have rank 4. '
  'Got array with shape: ' + str(x.shape))
@@ -1188,7 +1189,7 @@ def fit(self, x,
  if augment:
  ax = np.zeros(
  tuple([rounds * x.shape[0]] + list(x.shape)[1:]),
- dtype=keras.backend.floatx())
+ dtype=backend.floatx())
  for r in range(rounds):
  for i in range(x.shape[0]):
  ax[i + r * x.shape[0]] = self.random_transform(x[i])
@@ -1206,7 +1207,7 @@ def fit(self, x,
  broadcast_shape = [1, 1, 1]
  broadcast_shape[self.channel_axis - 1] = x.shape[self.channel_axis]
  self.std = np.reshape(self.std, broadcast_shape)
- x /= (self.std + keras.backend.epsilon())
+ x /= (self.std + backend.epsilon())
 
  if self.zca_whitening:
  flat_x = np.reshape(
@@ -1217,7 +1218,7 @@ def fit(self, x,
  self.principal_components = (u * s_inv).dot(u.T)
 
 
-class Iterator(keras.utils.Sequence):
+class Iterator(keras_utils.Sequence):
  """Base class for image data iterators.
 
  Every `Iterator` must implement the `_get_batches_of_transformed_samples`
@@ -1384,8 +1385,8 @@ def __init__(self, x, y, image_data_generator,
  if y is not None:
  y = y[split_idx:]
  if data_format is None:
- data_format = keras.backend.image_data_format()
- self.x = np.asarray(x, dtype=keras.backend.floatx())
+ data_format = backend.image_data_format()
+ self.x = np.asarray(x, dtype=backend.floatx())
  self.x_misc = x_misc
  if self.x.ndim != 4:
  raise ValueError('Input data in `NumpyArrayIterator` '
@@ -1421,12 +1422,12 @@ def __init__(self, x, y, image_data_generator,
 
  def _get_batches_of_transformed_samples(self, index_array):
  batch_x = np.zeros(tuple([len(index_array)] + list(self.x.shape)[1:]),
- dtype=keras.backend.floatx())
+ dtype=backend.floatx())
  for i, j in enumerate(index_array):
  x = self.x[j]
  params = self.image_data_generator.get_random_transform(x.shape)
  x = self.image_data_generator.apply_transform(
- x.astype(keras.backend.floatx()), params)
+ x.astype(backend.floatx()), params)
  x = self.image_data_generator.standardize(x)
  batch_x[i] = x
 
@@ -1625,7 +1626,7 @@ def __init__(self, directory, image_data_generator,
  subset=None,
  interpolation='nearest'):
  if data_format is None:
- data_format = keras.backend.image_data_format()
+ data_format = backend.image_data_format()
  self.directory = directory
  self.image_data_generator = image_data_generator
  self.target_size = tuple(target_size)
@@ -1722,7 +1723,7 @@ def __init__(self, directory, image_data_generator,
  def _get_batches_of_transformed_samples(self, index_array):
  batch_x = np.zeros(
  (len(index_array),) + self.image_shape,
- dtype=keras.backend.floatx())
+ dtype=backend.floatx())
  grayscale = self.color_mode == 'grayscale'
  # build batch of image data
  for i, j in enumerate(index_array):
@@ -1752,11 +1753,11 @@ def _get_batches_of_transformed_samples(self, index_array):
  elif self.class_mode == 'sparse':
  batch_y = self.classes[index_array]
  elif self.class_mode == 'binary':
- batch_y = self.classes[index_array].astype(keras.backend.floatx())
+ batch_y = self.classes[index_array].astype(backend.floatx())
  elif self.class_mode == 'categorical':
  batch_y = np.zeros(
  (len(batch_x), self.num_classes),
- dtype=keras.backend.floatx())
+ dtype=backend.floatx())
  for i, label in enumerate(self.classes[index_array]):
  batch_y[i, label] = 1.
  else:

keras_preprocessing/sequence.py

@@ -9,9 +9,9 @@
 import random
 from six.moves import range
 
-from . import keras_module
+from . import get_keras_submodule
 
-keras = keras_module()
+keras_utils = get_keras_submodule('utils')
 
 
 def pad_sequences(sequences, maxlen=None, dtype='int32',
@@ -250,7 +250,7 @@ def _remove_long_seq(maxlen, seq, label):
  return new_seq, new_label
 
 
-class TimeseriesGenerator(keras.utils.Sequence):
+class TimeseriesGenerator(keras_utils.Sequence):
  """Utility class for generating batches of temporal data.
 
  This class takes in a sequence of data-points gathered at

tests/image_test.py

@@ -5,6 +5,15 @@
 import tempfile
 import shutil
 
+import keras
+
+# TODO: remove the 3 lines below once the Keras release
+# is configured to use keras_preprocessing
+import keras_preprocessing
+keras_preprocessing.set_keras_submodules(
+ backend=keras.backend, utils=keras.utils)
+
+# This enables this import
 from keras_preprocessing import image
 
 

tests/integration_test.py

@@ -5,20 +5,31 @@
  from importlib import reload
 
 
-def test_dynamic_backend_setting():
+def test_that_internal_imports_are_not_overriden():
+ # Test that changing the keras module after importing
+ # Keras does not override keras.preprocessing's keras module
  import keras_preprocessing
  reload(keras_preprocessing)
- assert keras_preprocessing._KERAS_MODULE is None
- from tensorflow import keras as keras_ref
- keras_preprocessing.set_keras_module('tensorflow.keras')
- from keras_preprocessing import image
- assert image.keras_module() is keras_ref
-
- import keras as keras_ref
- keras_preprocessing.set_keras_module('keras')
- assert image.keras_module() is keras_ref
- reload(image)
- assert image.keras_module() is keras_ref
+ assert keras_preprocessing._KERAS_BACKEND is None
+
+ import keras
+ if not hasattr(keras.preprocessing.image, 'image'):
+ return # Old Keras, don't run.
+
+ import tensorflow as tf
+ keras_preprocessing.set_keras_submodules(backend=tf.keras.backend,
+ utils=tf.keras.utils)
+ assert keras.preprocessing.image.image.backend is keras.backend
+
+ # Now test the reverse order
+ del keras
+ reload(keras_preprocessing)
+ assert keras_preprocessing._KERAS_BACKEND is None
+
+ keras_preprocessing.set_keras_submodules(backend=tf.keras.backend,
+ utils=tf.keras.utils)
+ import keras
+ assert keras.preprocessing.image.image.backend is keras.backend
 
 
 if __name__ == '__main__':

tests/sequence_test.py

@@ -4,6 +4,14 @@
 from numpy.testing import assert_allclose
 from numpy.testing import assert_raises
 
+import keras
+
+# TODO: remove the 3 lines below once the Keras release
+# is configured to use keras_preprocessing
+import keras_preprocessing
+keras_preprocessing.set_keras_submodules(
+ backend=keras.backend, utils=keras.utils)
+
 from keras_preprocessing import sequence
 
 

tests/text_test.py

@@ -2,6 +2,14 @@
 import numpy as np
 import pytest
 
+import keras
+
+# TODO: remove the 3 lines below once the Keras release
+# is configured to use keras_preprocessing
+import keras_preprocessing
+keras_preprocessing.set_keras_submodules(
+ backend=keras.backend, utils=keras.utils)
+
 from keras_preprocessing import text