Skip to content

Commit 0b496a2

Browse files
committed
[Java] Default to US-ASCII rather than UTF-8 for char data type unless a character encoding is provided plus do not default to UTF-8 for variable length encoding when type is uint8 so that string accessor methods and character encoding methods are not generate for raw binary data.
1 parent 638e56d commit 0b496a2

File tree

9 files changed

+141
-103
lines changed

9 files changed

+141
-103
lines changed

sbe-benchmarks/src/main/resources/car.xml

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@
1717
<type name="blockLength" primitiveType="uint16"/>
1818
<type name="numInGroup" primitiveType="uint16"/>
1919
</composite>
20+
<composite name="varStringEncoding">
21+
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
22+
<type name="varData" primitiveType="uint8" length="0" characterEncoding="ISO-8859-1"/>
23+
</composite>
2024
<composite name="varDataEncoding">
21-
<type name="length" primitiveType="uint16"/>
22-
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
25+
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
26+
<type name="varData" primitiveType="uint8" length="0"/>
2327
</composite>
2428
</types>
2529
<types>
@@ -69,7 +73,7 @@
6973
<field name="seconds" id="16" type="float"/>
7074
</group>
7175
</group>
72-
<data name="manufacturer" id="17" type="varDataEncoding"/>
73-
<data name="model" id="18" type="varDataEncoding"/>
76+
<data name="manufacturer" id="17" type="varStringEncoding"/>
77+
<data name="model" id="18" type="varStringEncoding"/>
7478
</sbe:message>
7579
</sbe:messageSchema>

sbe-benchmarks/src/main/resources/fix-message-samples.xml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@
1515
-->
1616

1717
<types>
18-
<type name="string2" length="2" primitiveType="char"/>
19-
<type name="string3" length="3" primitiveType="char"/>
20-
<type name="string10" length="10" primitiveType="char"/>
21-
<type name="string12" length="12" primitiveType="char"/>
22-
<type name="string18" length="18" primitiveType="char"/>
23-
<type name="string20" length="20" primitiveType="char"/>
24-
<type name="string23" length="23" primitiveType="char"/>
25-
<type name="Symbol" length="6" primitiveType="char"/>
18+
<type name="string2" length="2" primitiveType="char" characterEncoding="US-ASCII"/>
19+
<type name="string3" length="3" primitiveType="char" characterEncoding="US-ASCII"/>
20+
<type name="string10" length="10" primitiveType="char" characterEncoding="US-ASCII"/>
21+
<type name="string12" length="12" primitiveType="char" characterEncoding="US-ASCII"/>
22+
<type name="string18" length="18" primitiveType="char" characterEncoding="US-ASCII"/>
23+
<type name="string20" length="20" primitiveType="char" characterEncoding="US-ASCII"/>
24+
<type name="string23" length="23" primitiveType="char" characterEncoding="US-ASCII"/>
25+
<type name="Symbol" length="6" primitiveType="char" characterEncoding="US-ASCII"/>
2626
<type name="timestamp" primitiveType="uint64" semanticType="UTCTimestamp"/>
2727
<type name="LocalMktDate" primitiveType="uint16" semanticType="LocalMktDate"/>
2828
<type name="optionalInt64" primitiveType="int64" presence="optional" semanticType="int"/>

sbe-samples/src/main/resources/example-extension-schema.xml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,14 @@
1818
<type name="blockLength" primitiveType="uint16"/>
1919
<type name="numInGroup" primitiveType="uint16"/>
2020
</composite>
21-
<composite name="varDataEncoding">
21+
<composite name="varStringEncoding">
2222
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
2323
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
2424
</composite>
25+
<composite name="varDataEncoding">
26+
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
27+
<type name="varData" primitiveType="uint8" length="0"/>
28+
</composite>
2529
</types>
2630
<types>
2731
<type name="ModelYear" primitiveType="uint16"/>
@@ -77,7 +81,7 @@
7781
<group name="fuelFigures" id="10" dimensionType="groupSizeEncoding">
7882
<field name="speed" id="11" type="uint16"/>
7983
<field name="mpg" id="12" type="float"/>
80-
<data name="usageDescription" id="200" type="varDataEncoding"/>
84+
<data name="usageDescription" id="200" type="varStringEncoding"/>
8185
</group>
8286
<group name="performanceFigures" id="13" dimensionType="groupSizeEncoding">
8387
<field name="octaneRating" id="14" type="Ron"/>
@@ -86,8 +90,8 @@
8690
<field name="seconds" id="17" type="float"/>
8791
</group>
8892
</group>
89-
<data name="manufacturer" id="18" type="varDataEncoding"/>
90-
<data name="model" id="19" type="varDataEncoding"/>
91-
<data name="activationCode" id="20" type="varDataEncoding"/>
93+
<data name="manufacturer" id="18" type="varStringEncoding"/>
94+
<data name="model" id="19" type="varStringEncoding"/>
95+
<data name="activationCode" id="20" type="varStringEncoding"/>
9296
</sbe:message>
9397
</sbe:messageSchema>

sbe-samples/src/main/resources/example-schema.xml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@
1717
<type name="blockLength" primitiveType="uint16"/>
1818
<type name="numInGroup" primitiveType="uint16"/>
1919
</composite>
20-
<composite name="varDataEncoding">
20+
<composite name="varStringEncoding">
2121
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
2222
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
2323
</composite>
24+
<composite name="varDataEncoding">
25+
<type name="length" primitiveType="uint32" maxValue="1073741824"/>
26+
<type name="varData" primitiveType="uint8" length="0"/>
27+
</composite>
2428
</types>
2529
<types>
2630
<type name="ModelYear" primitiveType="uint16"/>
@@ -75,7 +79,7 @@
7579
<group name="fuelFigures" id="10" dimensionType="groupSizeEncoding">
7680
<field name="speed" id="11" type="uint16"/>
7781
<field name="mpg" id="12" type="float"/>
78-
<data name="usageDescription" id="200" type="varDataEncoding"/>
82+
<data name="usageDescription" id="200" type="varStringEncoding"/>
7983
</group>
8084
<group name="performanceFigures" id="13" dimensionType="groupSizeEncoding">
8185
<field name="octaneRating" id="14" type="Ron"/>
@@ -84,8 +88,8 @@
8488
<field name="seconds" id="17" type="float"/>
8589
</group>
8690
</group>
87-
<data name="manufacturer" id="18" type="varDataEncoding"/>
88-
<data name="model" id="19" type="varDataEncoding"/>
89-
<data name="activationCode" id="20" type="varDataEncoding"/>
91+
<data name="manufacturer" id="18" type="varStringEncoding"/>
92+
<data name="model" id="19" type="varStringEncoding"/>
93+
<data name="activationCode" id="20" type="varStringEncoding"/>
9094
</sbe:message>
9195
</sbe:messageSchema>

sbe-tool/src/main/java/uk/co/real_logic/sbe/generation/java/JavaGenerator.java

Lines changed: 86 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -770,33 +770,36 @@ private void generateDataDecodeMethods(
770770
byteOrderStr,
771771
indent);
772772

773-
sb.append(String.format(
774-
"\n" +
775-
indent + " public String %1$s()\n" +
776-
indent + " {\n" +
777-
"%2$s" +
778-
indent + " final int headerLength = %3$d;\n" +
779-
indent + " final int limit = parentMessage.limit();\n" +
780-
indent + " final int dataLength = (int)%4$s;\n" +
781-
indent + " parentMessage.limit(limit + headerLength + dataLength);\n" +
782-
indent + " final byte[] tmp = new byte[dataLength];\n" +
783-
indent + " buffer.getBytes(limit + headerLength, tmp, 0, dataLength);\n\n" +
784-
indent + " final String value;\n" +
785-
indent + " try\n" +
786-
indent + " {\n" +
787-
indent + " value = new String(tmp, \"%5$s\");\n" +
788-
indent + " }\n" +
789-
indent + " catch (final java.io.UnsupportedEncodingException ex)\n" +
790-
indent + " {\n" +
791-
indent + " throw new RuntimeException(ex);\n" +
792-
indent + " }\n\n" +
793-
indent + " return value;\n" +
794-
indent + " }\n",
795-
formatPropertyName(propertyName),
796-
generateStringNotPresentCondition(token.version(), indent),
797-
sizeOfLengthField,
798-
generateGet(lengthType, "limit", byteOrderStr),
799-
characterEncoding));
773+
if (null != characterEncoding)
774+
{
775+
sb.append(String.format(
776+
"\n" +
777+
indent + " public String %1$s()\n" +
778+
indent + " {\n" +
779+
"%2$s" +
780+
indent + " final int headerLength = %3$d;\n" +
781+
indent + " final int limit = parentMessage.limit();\n" +
782+
indent + " final int dataLength = (int)%4$s;\n" +
783+
indent + " parentMessage.limit(limit + headerLength + dataLength);\n" +
784+
indent + " final byte[] tmp = new byte[dataLength];\n" +
785+
indent + " buffer.getBytes(limit + headerLength, tmp, 0, dataLength);\n\n" +
786+
indent + " final String value;\n" +
787+
indent + " try\n" +
788+
indent + " {\n" +
789+
indent + " value = new String(tmp, \"%5$s\");\n" +
790+
indent + " }\n" +
791+
indent + " catch (final java.io.UnsupportedEncodingException ex)\n" +
792+
indent + " {\n" +
793+
indent + " throw new RuntimeException(ex);\n" +
794+
indent + " }\n\n" +
795+
indent + " return value;\n" +
796+
indent + " }\n",
797+
formatPropertyName(propertyName),
798+
generateStringNotPresentCondition(token.version(), indent),
799+
sizeOfLengthField,
800+
generateGet(lengthType, "limit", byteOrderStr),
801+
characterEncoding));
802+
}
800803
}
801804

802805
private void generateDataEncodeMethods(
@@ -832,37 +835,41 @@ private void generateDataEncodeMethods(
832835
byteOrderStr,
833836
indent);
834837

835-
sb.append(String.format(
836-
"\n" +
837-
indent + " public %1$s %2$s(final String value)\n" +
838-
indent + " {\n" +
839-
indent + " final byte[] bytes;\n" +
840-
indent + " try\n" +
841-
indent + " {\n" +
842-
indent + " bytes = value.getBytes(\"%3$s\");\n" +
843-
indent + " }\n" +
844-
indent + " catch (final java.io.UnsupportedEncodingException ex)\n" +
845-
indent + " {\n" +
846-
indent + " throw new RuntimeException(ex);\n" +
847-
indent + " }\n\n" +
848-
indent + " final int length = bytes.length;\n" +
849-
indent + " if (length > %4$d)\n" +
850-
indent + " {\n" +
851-
indent + " throw new IllegalArgumentException(\"length > max value for type: \" + length);\n" +
852-
indent + " }\n\n" +
853-
indent + " final int headerLength = %5$d;\n" +
854-
indent + " final int limit = parentMessage.limit();\n" +
855-
indent + " parentMessage.limit(limit + headerLength + length);\n" +
856-
indent + " %6$s;\n" +
857-
indent + " buffer.putBytes(limit + headerLength, bytes, 0, length);\n\n" +
858-
indent + " return this;\n" +
859-
indent + " }\n",
860-
className,
861-
formatPropertyName(propertyName),
862-
characterEncoding,
863-
maxLengthValue,
864-
sizeOfLengthField,
865-
generatePut(lengthType, "limit", "length", byteOrderStr)));
838+
if (null != characterEncoding)
839+
{
840+
sb.append(String.format(
841+
"\n" +
842+
indent + " public %1$s %2$s(final String value)\n" +
843+
indent + " {\n" +
844+
indent + " final byte[] bytes;\n" +
845+
indent + " try\n" +
846+
indent + " {\n" +
847+
indent + " bytes = value.getBytes(\"%3$s\");\n" +
848+
indent + " }\n" +
849+
indent + " catch (final java.io.UnsupportedEncodingException ex)\n" +
850+
indent + " {\n" +
851+
indent + " throw new RuntimeException(ex);\n" +
852+
indent + " }\n\n" +
853+
indent + " final int length = bytes.length;\n" +
854+
indent + " if (length > %4$d)\n" +
855+
indent + " {\n" +
856+
indent + " throw new IllegalArgumentException(" +
857+
"\"length > max value for type: \" + length);\n" +
858+
indent + " }\n\n" +
859+
indent + " final int headerLength = %5$d;\n" +
860+
indent + " final int limit = parentMessage.limit();\n" +
861+
indent + " parentMessage.limit(limit + headerLength + length);\n" +
862+
indent + " %6$s;\n" +
863+
indent + " buffer.putBytes(limit + headerLength, bytes, 0, length);\n\n" +
864+
indent + " return this;\n" +
865+
indent + " }\n",
866+
className,
867+
formatPropertyName(propertyName),
868+
characterEncoding,
869+
maxLengthValue,
870+
sizeOfLengthField,
871+
generatePut(lengthType, "limit", "length", byteOrderStr)));
872+
}
866873
}
867874

868875
private void generateDataTypedDecoder(
@@ -1787,16 +1794,19 @@ private static int sizeOfPrimitive(final Encoding encoding)
17871794
}
17881795

17891796
private static void generateCharacterEncodingMethod(
1790-
final StringBuilder sb, final String propertyName, final String encoding, final String indent)
1797+
final StringBuilder sb, final String propertyName, final String characterEncoding, final String indent)
17911798
{
1792-
sb.append(String.format(
1793-
"\n" +
1794-
indent + " public static String %sCharacterEncoding()\n" +
1795-
indent + " {\n" +
1796-
indent + " return \"%s\";\n" +
1797-
indent + " }\n",
1798-
formatPropertyName(propertyName),
1799-
encoding));
1799+
if (null != characterEncoding)
1800+
{
1801+
sb.append(String.format(
1802+
"\n" +
1803+
indent + " public static String %sCharacterEncoding()\n" +
1804+
indent + " {\n" +
1805+
indent + " return \"%s\";\n" +
1806+
indent + " }\n",
1807+
formatPropertyName(propertyName),
1808+
characterEncoding));
1809+
}
18001810
}
18011811

18021812
private CharSequence generateConstPropertyMethods(
@@ -2791,9 +2801,17 @@ private StringBuilder appendDecoderDisplay(
27912801

27922802
append(sb, indent, "//" + varDataToken);
27932803

2804+
final String characterEncoding = varData.get(i + 3).encoding().characterEncoding();
27942805
final String varDataName = formatPropertyName(varDataToken.name());
27952806
append(sb, indent, "builder.append(\"" + varDataName + Separators.KEY_VALUE + "\");");
2796-
append(sb, indent, "builder.append(" + varDataName + "());");
2807+
if (null == characterEncoding)
2808+
{
2809+
append(sb, indent, "builder.append(" + varDataName + "Length() + \" raw bytes\");");
2810+
}
2811+
else
2812+
{
2813+
append(sb, indent, "builder.append(" + varDataName + "());");
2814+
}
27972815

27982816
lengthBeforeLastGeneratedSeparator = sb.length();
27992817
Separators.FIELD.appendToGeneratedBuilder(sb, indent, "builder");

sbe-tool/src/main/java/uk/co/real_logic/sbe/xml/EncodedDataType.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,15 @@ public EncodedDataType(final Node node, final String givenName)
6666
primitiveType = PrimitiveType.get(getAttributeValue(node, "primitiveType"));
6767
length = Integer.parseInt(getAttributeValue(node, "length", "1"));
6868
varLen = Boolean.parseBoolean(getAttributeValue(node, "variableLength", "false"));
69-
characterEncoding = getAttributeValue(node, "characterEncoding", "UTF-8");
69+
70+
if (PrimitiveType.CHAR == primitiveType)
71+
{
72+
characterEncoding = getAttributeValue(node, "characterEncoding", "US-ASCII");
73+
}
74+
else
75+
{
76+
characterEncoding = getAttributeValueOrNull(node, "characterEncoding");
77+
}
7078

7179
if (presence() == CONSTANT)
7280
{

sbe-tool/src/test/resources/code-generation-schema.xml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
<type name="blockLength" primitiveType="uint16"/>
2727
<type name="numInGroup" primitiveType="uint16"/>
2828
</composite>
29-
<composite name="varDataEncoding">
29+
<composite name="varStringEncoding">
3030
<type name="length" primitiveType="uint16"/>
3131
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
3232
</composite>
@@ -72,7 +72,7 @@
7272
<group name="fuelFigures" id="10" dimensionType="groupSizeEncoding">
7373
<field name="speed" id="11" type="uint16"/>
7474
<field name="mpg" id="12" type="float"/>
75-
<data name="usageDescription" id="200" type="varDataEncoding"/>
75+
<data name="usageDescription" id="200" type="varStringEncoding"/>
7676
</group>
7777
<group name="performanceFigures" id="13" dimensionType="groupSizeEncoding">
7878
<field name="octaneRating" id="14" type="Ron"/>
@@ -81,8 +81,8 @@
8181
<field name="seconds" id="17" type="float"/>
8282
</group>
8383
</group>
84-
<data name="manufacturer" id="18" type="varDataEncoding"/>
85-
<data name="model" id="19" type="varDataEncoding"/>
86-
<data name="activationCode" id="20" type="varDataEncoding"/>
84+
<data name="manufacturer" id="18" type="varStringEncoding"/>
85+
<data name="model" id="19" type="varStringEncoding"/>
86+
<data name="activationCode" id="20" type="varStringEncoding"/>
8787
</sbe:message>
8888
</sbe:messageSchema>

sbe-tool/src/test/resources/error-handler-invalid-name.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
<type name="blockLength" primitiveType="uint16"/>
1616
<type name="numInGroup" primitiveType="uint8"/>
1717
</composite>
18-
<composite name="varDataEncoding">
18+
<composite name="varStringEncoding">
1919
<type name="length" primitiveType="uint8"/>
2020
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
2121
</composite>
@@ -32,7 +32,7 @@
3232
<field name="field1" id="6" type="uint16"/>
3333
<field name="field2" id="7" type="float"/>
3434
</group>
35-
<data name="invalid data field name!" id="8" type="varDataEncoding"/>
36-
<data name="validDataFieldName" id="9" type="varDataEncoding"/>
35+
<data name="invalid data field name!" id="8" type="varStringEncoding"/>
36+
<data name="validDataFieldName" id="9" type="varStringEncoding"/>
3737
</message>
3838
</messageSchema>

sbe-tool/src/test/resources/json-printer-test-schema.xml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
<type name="blockLength" primitiveType="uint16"/>
1818
<type name="numInGroup" primitiveType="uint8"/>
1919
</composite>
20-
<composite name="varDataEncoding">
20+
<composite name="varStringEncoding">
2121
<type name="length" primitiveType="uint8"/>
2222
<type name="varData" primitiveType="uint8" length="0" characterEncoding="UTF-8"/>
2323
</composite>
@@ -69,8 +69,8 @@
6969
<field name="seconds" id="16" type="float"/>
7070
</group>
7171
</group>
72-
<data name="manufacturer" id="17" type="varDataEncoding"/>
73-
<data name="model" id="18" type="varDataEncoding"/>
74-
<data name="activationCode" id="19" type="varDataEncoding"/>
72+
<data name="manufacturer" id="17" type="varStringEncoding"/>
73+
<data name="model" id="18" type="varStringEncoding"/>
74+
<data name="activationCode" id="19" type="varStringEncoding"/>
7575
</sbe:message>
7676
</sbe:messageSchema>

0 commit comments

Comments
 (0)