[Java] Add support for transforming the schema to generate code for older versions.

Author: mikeb01

.gitignore

@@ -113,3 +113,5 @@ rust/Cargo.lock
 # Mac
 .DS_Store
 /sbe-tool/src/main/golang/uk_co_real_logic_sbe_ir_generated/
+
+/generated/

sbe-tool/src/main/java/uk/co/real_logic/sbe/SbeTool.java

@@ -18,12 +18,20 @@
 import org.agrona.DirectBuffer;
 import org.agrona.MutableDirectBuffer;
 import org.xml.sax.InputSource;
-import uk.co.real_logic.sbe.generation.*;
-import uk.co.real_logic.sbe.ir.*;
+import uk.co.real_logic.sbe.generation.CodeGenerator;
+import uk.co.real_logic.sbe.generation.TargetCodeGenerator;
+import uk.co.real_logic.sbe.generation.TargetCodeGeneratorLoader;
+import uk.co.real_logic.sbe.ir.Ir;
+import uk.co.real_logic.sbe.ir.IrDecoder;
+import uk.co.real_logic.sbe.ir.IrEncoder;
 import uk.co.real_logic.sbe.xml.*;
 
-import java.io.*;
-import java.nio.file.*;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * A tool for running the SBE parser, validator, and code generator.
@@ -176,6 +184,17 @@ public class SbeTool
  */
  public static final String DECODE_UNKNOWN_ENUM_VALUES = "sbe.decode.unknown.enum.values";
 
+ /**
+ * Configuration option used to manage sinceVersion based transformations. When set, parsed schemas will be
+ * transformed to discard messages and types higher than the specified version. This can be useful when needing
+ * to generate older versions of a schema to do version compatibility testing.
+ * <p>
+ * This field can contain a list of ordered pairs in the form:
+ * <code>((&lt;schema id&gt; | '*') ':' &lt;schema id&gt;)(',' ((&lt;schema id&gt; | '*') ':' &lt;schema id&gt;))*</code>.
+ * E.g. <code>123:5,*:6</code> which means transform schema with id = 123 to version 5, all others to version 6.
+ */
+ public static final String SCHEMA_TRANSFORM_VERSION = "sbe.schema.transform.version";
+
  /**
  * Main entry point for the SBE Tool.
  *
@@ -201,7 +220,10 @@ public static void main(final String[] args) throws Exception
  validateAgainstSchema(fileName, xsdFilename);
  }
 
- ir = new IrGenerator().generate(parseSchema(fileName), System.getProperty(TARGET_NAMESPACE));
+ final MessageSchema schema = parseSchema(fileName);
+ final SchemaTransformer transformer = new SchemaTransformerFactory(
+ System.getProperty(SCHEMA_TRANSFORM_VERSION));
+ ir = new IrGenerator().generate(transformer.transform(schema), System.getProperty(TARGET_NAMESPACE));
  }
  else if (fileName.endsWith(".sbeir"))
  {

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/IdKeyedSchemaTransformer.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.Map;
+
+class IdKeyedSchemaTransformer implements SchemaTransformer
+{
+ private final Map<Integer, SchemaTransformer> transformerBySchemaId;
+ private final SchemaTransformer defaultTransformer;
+
+ IdKeyedSchemaTransformer(
+ final Map<Integer, SchemaTransformer> transformerBySchemaId,
+ final SchemaTransformer defaultTransformer)
+ {
+ this.transformerBySchemaId = transformerBySchemaId;
+ this.defaultTransformer = defaultTransformer;
+ }
+
+ public MessageSchema transform(final MessageSchema originalSchema)
+ {
+ return lookupTransformer(originalSchema.id()).transform(originalSchema);
+ }
+
+ SchemaTransformer lookupTransformer(final int schemaId)
+ {
+ return transformerBySchemaId.getOrDefault(schemaId, defaultTransformer);
+ }
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/MessageSchema.java

@@ -15,8 +15,8 @@
  */
 package uk.co.real_logic.sbe.xml;
 
-import org.w3c.dom.Node;
 import org.agrona.Verify;
+import org.w3c.dom.Node;
 
 import java.nio.ByteOrder;
 import java.util.*;
@@ -62,6 +62,28 @@ public class MessageSchema
  ((CompositeType)typeByNameMap.get(this.headerType)).checkForWellFormedMessageHeader(schemaNode);
  }
 
+ MessageSchema(
+ final String packageName,
+ final String description,
+ final int id,
+ final int version,
+ final String semanticVersion,
+ final ByteOrder byteOrder,
+ final String headerType,
+ final Map<String, Type> typeByNameMap,
+ final Map<Long, Message> messageByIdMap)
+ {
+ this.packageName = packageName;
+ this.description = description;
+ this.id = id;
+ this.version = version;
+ this.semanticVersion = semanticVersion;
+ this.byteOrder = byteOrder;
+ this.headerType = headerType;
+ this.typeByNameMap = typeByNameMap;
+ this.messageByIdMap = messageByIdMap;
+ }
+
  /**
  * The Schema headerType for message headers. This should be a {@link CompositeType}.
  *
@@ -154,6 +176,16 @@ public Collection<Message> messages()
  return messageByIdMap.values();
  }
 
+ /**
+ * Get the {@link Collection} of {@link Type}s for this Schema.
+ *
+ * @return the {@link Collection} of {@link Type}s for this Schema.
+ */
+ public Collection<Type> types()
+ {
+ return typeByNameMap.values();
+ }
+
  /**
  * Return the byte order specified by the messageSchema
  *

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SchemaTransformer.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+/**
+ * An abstraction that allows for general transformations of the IR.
+ */
+public interface SchemaTransformer
+{
+ /**
+ * Does an identity transform of the incoming schema.
+ */
+ SchemaTransformer IDENTITY_TRANSFORMER = originalSchema -> originalSchema;
+
+ /**
+ * Return a new MessageSchema that has been transformed from the original.
+ *
+ * @param originalSchema to be transformed
+ * @return resulting transformed schema.
+ */
+ MessageSchema transform(final MessageSchema originalSchema);
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SchemaTransformerFactory.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Entry point for schema transformations, will check all incoming configuration/properties and create the appropriate
+ * transformers as required.
+ */
+public class SchemaTransformerFactory implements SchemaTransformer
+{
+ private final SchemaTransformer transformer;
+
+ public SchemaTransformerFactory(final String schemaTransformConfig)
+ {
+ transformer = parse(schemaTransformConfig);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public MessageSchema transform(final MessageSchema originalSchema)
+ {
+ return transformer.transform(originalSchema);
+ }
+
+ static SchemaTransformer parse(final String configuration)
+ {
+ if (null == configuration || configuration.isEmpty())
+ {
+ return IDENTITY_TRANSFORMER;
+ }
+
+ final String[] split = configuration.split(",");
+ if (0 == split.length)
+ {
+ return IDENTITY_TRANSFORMER;
+ }
+
+ final HashMap<Integer, SchemaTransformer> transformerBySchemaId = new HashMap<>();
+ parseComponents(split, transformerBySchemaId);
+
+ SchemaTransformer defaultTransformer = transformerBySchemaId.remove(-1);
+ defaultTransformer = null != defaultTransformer ? defaultTransformer : IDENTITY_TRANSFORMER;
+
+ return transformerBySchemaId.isEmpty() ?
+ defaultTransformer : new IdKeyedSchemaTransformer(transformerBySchemaId, defaultTransformer);
+ }
+
+ private static void parseComponents(
+ final String[] configuration,
+ final Map<Integer, SchemaTransformer> transformerBySchemaId)
+ {
+ for (String field : configuration)
+ {
+ final String[] fieldParts = field.split(":");
+
+ if (2 != fieldParts.length)
+ {
+ throw new IllegalArgumentException("version transformation property part '" + field + "' is invalid");
+ }
+
+ final int schemaId = "*".equals(fieldParts[0]) ? -1 : Integer.parseInt(fieldParts[0].trim());
+ final int sinceVersion = Integer.parseInt(fieldParts[1].trim());
+ transformerBySchemaId.put(schemaId, new SinceVersionSchemaTransformer(sinceVersion));
+ }
+ }
+
+ SchemaTransformer delegate()
+ {
+ return transformer;
+ }
+}

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/SinceVersionSchemaTransformer.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013-2022 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package uk.co.real_logic.sbe.xml;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+class SinceVersionSchemaTransformer implements SchemaTransformer
+{
+ private final int sinceVersion;
+
+ public SinceVersionSchemaTransformer(final int sinceVersion)
+ {
+ this.sinceVersion = sinceVersion;
+ }
+
+ public MessageSchema transform(final MessageSchema originalSchema)
+ {
+ final Collection<Type> types = originalSchema.types();
+ final Map<String, Type> newTypes = new HashMap<>();
+
+ for (Type type : types)
+ {
+ if (type.sinceVersion() <= this.sinceVersion)
+ {
+ newTypes.put(type.name(), type);
+ }
+ }
+
+ final Collection<Message> messages = originalSchema.messages();
+ final Map<Long, Message> newMessages = new HashMap<>();
+ for (Message message : messages)
+ {
+ if (message.sinceVersion() <= this.sinceVersion)
+ {
+ newMessages.put((long)message.id(), message);
+ }
+ }
+
+ return new MessageSchema(
+ originalSchema.packageName(),
+ originalSchema.description(),
+ originalSchema.id(),
+ sinceVersion,
+ originalSchema.semanticVersion(),
+ originalSchema.byteOrder(),
+ originalSchema.messageHeader().name(),
+ newTypes,
+ newMessages);
+ }
+
+ int sinceVersion()
+ {
+ return sinceVersion;
+ }
+}