DATAREDIS-1091 - Add configuration to disable storage of shadow copies for entities using TimeToLive.

Original pull request: spring-projects#517.

Author: christophstrobl

src/main/asciidoc/reference/redis-repositories.adoc

@@ -591,6 +591,8 @@ NOTE: The keyspace notification message listener alters `notify-keyspace-events`
 
 NOTE: Redis Pub/Sub messages are not persistent. If a key expires while the application is down, the expiry event is not processed, which may lead to secondary indexes containing references to the expired object.
 
+NOTE: `@EnableKeyspaceEvents(shadowCopy = OFF)` disable storage of phantom copies and reduces data size within Redis. `RedisKeyExpiredEvent` will only contain the `id` of the expired key.
+
 [[redis.repositories.references]]
 == Persisting References
 Marking properties with `@Reference` allows storing a simple key reference instead of copying values into the hash itself.

src/main/java/org/springframework/data/redis/core/RedisKeyValueAdapter.java

@@ -117,6 +117,7 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
 
 private EnableKeyspaceEvents enableKeyspaceEvents = EnableKeyspaceEvents.OFF;
 private @Nullable String keyspaceNotificationsConfigParameter = null;
+private ShadowCopy shadowCopy = ShadowCopy.DEFAULT;
 
 /**
  * Creates new {@link RedisKeyValueAdapter} with default {@link RedisMappingContext} and default
@@ -238,11 +239,13 @@ public Object put(Object id, Object item, String keyspace) {
 
 connection.expire(objectKey, rdo.getTimeToLive());
 
-// add phantom key so values can be restored
-byte[] phantomKey = ByteUtils.concat(objectKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
-connection.del(phantomKey);
-connection.hMSet(phantomKey, rdo.getBucket().rawMap());
-connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
+if (keepShadowCopy()) { // add phantom key so values can be restored
+
+byte[] phantomKey = ByteUtils.concat(objectKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
+connection.del(phantomKey);
+connection.hMSet(phantomKey, rdo.getBucket().rawMap());
+connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
+}
 }
 
 connection.sAdd(toBytes(rdo.getKeyspace()), key);
@@ -475,10 +478,12 @@ public void update(PartialUpdate<?> update) {
 
 connection.expire(redisKey, rdo.getTimeToLive());
 
-// add phantom key so values can be restored
-byte[] phantomKey = ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
-connection.hMSet(phantomKey, rdo.getBucket().rawMap());
-connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
+if (keepShadowCopy()) { // add phantom key so values can be restored
+
+byte[] phantomKey = ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
+connection.hMSet(phantomKey, rdo.getBucket().rawMap());
+connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
+}
 
 } else {
 
@@ -661,6 +666,16 @@ public void setKeyspaceNotificationsConfigParameter(String keyspaceNotifications
 this.keyspaceNotificationsConfigParameter = keyspaceNotificationsConfigParameter;
 }
 
+/**
+ * Configure storage of phantom keys (shadow copies) of expiring entities.
+ *
+ * @param shadowCopy must not be {@literal null}.
+ * @since 2.3
+ */
+public void setShadowCopy(ShadowCopy shadowCopy) {
+this.shadowCopy = shadowCopy;
+}
+
 /**
  * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
  * @since 1.8
@@ -773,7 +788,8 @@ public void onMessage(Message message, @Nullable byte[] pattern) {
 
 byte[] key = message.getBody();
 
-byte[] phantomKey = ByteUtils.concat(key, converter.getConversionService().convert(KeyspaceIdentifier.PHANTOM_SUFFIX, byte[].class));
+byte[] phantomKey = ByteUtils.concat(key,
+converter.getConversionService().convert(KeyspaceIdentifier.PHANTOM_SUFFIX, byte[].class));
 
 Map<byte[], byte[]> hash = ops.execute((RedisCallback<Map<byte[], byte[]>>) connection -> {
 
@@ -789,7 +805,8 @@ public void onMessage(Message message, @Nullable byte[] pattern) {
 Object value = converter.read(Object.class, new RedisData(hash));
 
 String channel = !ObjectUtils.isEmpty(message.getChannel())
-? converter.getConversionService().convert(message.getChannel(), String.class) : null;
+? converter.getConversionService().convert(message.getChannel(), String.class)
+: null;
 
 RedisKeyExpiredEvent event = new RedisKeyExpiredEvent(channel, key, value);
 
@@ -813,6 +830,18 @@ private boolean isKeyExpirationMessage(Message message) {
 }
 }
 
+private boolean keepShadowCopy() {
+
+switch (shadowCopy) {
+case OFF:
+return false;
+case ON:
+return true;
+default:
+return this.expirationListener.get() != null;
+}
+}
+
 /**
  * @author Christoph Strobl
  * @since 1.8
@@ -835,6 +864,31 @@ public enum EnableKeyspaceEvents {
 OFF
 }
 
+/**
+ * Configuration flag controlling storage of phantom keys (shadow copies) of expiring entities to read them later when
+ * publishing {@link RedisKeyspaceEvent}.
+ *
+ * @author Christoph Strobl
+ * @since 2.3
+ */
+public enum ShadowCopy {
+
+/**
+ * Store shadow copies of expiring entities depending on the {@link EnableKeyspaceEvents}.
+ */
+DEFAULT,
+
+/**
+ * Store shadow copies of expiring entities.
+ */
+ON,
+
+/**
+ * Do not store shadow copies.
+ */
+OFF
+}
+
 /**
  * Container holding update information like fields to remove from the Redis Hash.
  *

src/main/java/org/springframework/data/redis/repository/configuration/EnableRedisRepositories.java

@@ -28,6 +28,7 @@
 import org.springframework.data.keyvalue.core.KeyValueOperations;
 import org.springframework.data.keyvalue.repository.config.QueryCreatorType;
 import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
+import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
 import org.springframework.data.redis.core.RedisOperations;
 import org.springframework.data.redis.core.convert.KeyspaceConfiguration;
 import org.springframework.data.redis.core.index.IndexConfiguration;
@@ -166,6 +167,15 @@
  */
 EnableKeyspaceEvents enableKeyspaceEvents() default EnableKeyspaceEvents.OFF;
 
+/**
+ * Configuration flag controlling storage of phantom keys (shadow copies) of expiring entities to read them later when
+ * publishing {@link org.springframework.data.redis.core.RedisKeyspaceEvent keyspace events}.
+ * 
+ * @return
+ * @since 2.3
+ */
+ShadowCopy shadowCopy() default ShadowCopy.DEFAULT;
+
 /**
  * Configure the {@literal notify-keyspace-events} property if not already set. <br />
  * Use an empty {@link String} to keep (<b>not</b> alter) existing server configuration.

src/main/java/org/springframework/data/redis/repository/configuration/RedisRepositoryConfigurationExtension.java

@@ -28,6 +28,7 @@
 import org.springframework.data.redis.core.RedisHash;
 import org.springframework.data.redis.core.RedisKeyValueAdapter;
 import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
+import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
 import org.springframework.data.redis.core.RedisKeyValueTemplate;
 import org.springframework.data.redis.core.convert.MappingConfiguration;
 import org.springframework.data.redis.core.convert.MappingRedisConverter;
@@ -145,6 +146,8 @@ private static AbstractBeanDefinition createRedisKeyValueAdapter(RepositoryConfi
 configuration.getRequiredAttribute("enableKeyspaceEvents", EnableKeyspaceEvents.class)) //
 .addPropertyValue("keyspaceNotificationsConfigParameter",
 configuration.getAttribute("keyspaceNotificationsConfigParameter", String.class).orElse("")) //
+.addPropertyValue("shadowCopy",
+configuration.getRequiredAttribute("shadowCopy", ShadowCopy.class)) //
 .getBeanDefinition();
 }
 

src/test/java/org/springframework/data/redis/core/RedisKeyValueAdapterTests.java

@@ -37,7 +37,6 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
-
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.data.annotation.Id;
 import org.springframework.data.annotation.Reference;
@@ -50,6 +49,7 @@
 import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
 import org.springframework.data.redis.connection.lettuce.LettuceTestClientResources;
 import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
+import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
 import org.springframework.data.redis.core.convert.KeyspaceConfiguration;
 import org.springframework.data.redis.core.convert.MappingConfiguration;
 import org.springframework.data.redis.core.index.GeoIndexed;
@@ -341,7 +341,6 @@ public void keyExpiredEventShouldRemoveHelperStructures() throws Exception {
 assertThat(template.hasKey("persons:firstname:rand")).isFalse();
 assertThat(template.hasKey("persons:1:idx")).isFalse();
 assertThat(template.opsForSet().members("persons")).doesNotContain("1");
-;
 }
 
 @Test // DATAREDIS-744
@@ -687,6 +686,56 @@ public void updateShouldAlterGeoIndexCorrectlyOnUpdate() {
 assertThat(updatedLocation.getY()).isCloseTo(18D, offset(0.005));
 }
 
+@Test // DATAREDIS-1091
+public void phantomKeyNotInsertedOnPutWhenShadowCopyIsTurnedOff() {
+
+RedisMappingContext mappingContext = new RedisMappingContext(
+new MappingConfiguration(new IndexConfiguration(), new KeyspaceConfiguration()));
+mappingContext.afterPropertiesSet();
+
+RedisKeyValueAdapter kvAdapter = new RedisKeyValueAdapter(template, mappingContext);
+kvAdapter.setShadowCopy(ShadowCopy.OFF);
+
+ExpiringPerson rand = new ExpiringPerson();
+rand.age = 24;
+rand.ttl = 3000L;
+
+kvAdapter.put("1", rand, "persons");
+
+assertThat(template.hasKey("persons:1:phantom")).isFalse();
+}
+
+@Test // DATAREDIS-1091
+public void phantomKeyInsertedOnPutWhenShadowCopyIsTurnedOn() {
+
+RedisMappingContext mappingContext = new RedisMappingContext(
+new MappingConfiguration(new IndexConfiguration(), new KeyspaceConfiguration()));
+mappingContext.afterPropertiesSet();
+
+RedisKeyValueAdapter kvAdapter = new RedisKeyValueAdapter(template, mappingContext);
+kvAdapter.setShadowCopy(ShadowCopy.ON);
+
+ExpiringPerson rand = new ExpiringPerson();
+rand.age = 24;
+rand.ttl = 3000L;
+
+kvAdapter.put("1", rand, "persons");
+
+assertThat(template.hasKey("persons:1:phantom")).isTrue();
+}
+
+@Test // DATAREDIS-1091
+public void phantomKeyInsertedOnPutWhenShadowCopyIsInDefaultAndKeyspaceNotificationEnabled() {
+
+ExpiringPerson rand = new ExpiringPerson();
+rand.age = 24;
+rand.ttl = 3000L;
+
+adapter.put("1", rand, "persons");
+
+assertThat(template.hasKey("persons:1:phantom")).isTrue();
+}
+
 /**
  * Wait up to 5 seconds until {@code key} is no longer available in Redis.
  *
@@ -776,6 +825,11 @@ static class TaVeren extends Person {
 
 }
 
+static class ExpiringPerson extends Person {
+
+@TimeToLive Long ttl;
+}
+
 @KeySpace("locations")
 static class Location {