Skip to content

Commit f44b39f

Browse files
mjpt777mjpt777
authored
Merge pull request aeron-io#667 from feribg/python
[Java] Python generator implementation aeron-io#665
2 parents eea5d96 + eb65fdd commit f44b39f

File tree

12 files changed

+3851
-0
lines changed

12 files changed

+3851
-0
lines changed

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,18 @@ Users of CSharp generated code should see the [user documentation](https://githu
118118
Developers wishing to enhance the CSharp generator should see the [developer documentation](https://github.com/real-logic/simple-binary-encoding/blob/master/csharp/README.md)
119119

120120

121+
Python Build
122+
------------
123+
Python support is experimental and tested against Python >= 3.6. Other python versions and implementations
124+
should also work, given that they have support for the `struct` package and the `buffer protocol`, but they aren't
125+
officially supported.
126+
127+
Use the following the build the java stub generator and execute the python tests:
128+
```bash
129+
$ ./gradlew
130+
$ ./gradlew runPythonTests
131+
```
132+
121133
License (See LICENSE file for full license)
122134
-------------------------------------------
123135
Copyright 2013-2019 Real Logic Limited

‎build.gradle‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -775,6 +775,84 @@ task uploadToMavenCentral {
775775
'sbe-all:uploadShadow'
776776
}
777777

778+
/**
779+
* Python tests and benchmarks
780+
*/
781+
task generatePythonCarExample(type: JavaExec) {
782+
main = 'uk.co.real_logic.sbe.SbeTool'
783+
classpath = project(':sbe-all').sourceSets.main.runtimeClasspath
784+
systemProperties(
785+
'sbe.output.dir': 'python/tests',
786+
'sbe.target.language': 'uk.co.real_logic.sbe.generation.python.Python',
787+
'sbe.xinclude.aware': 'true',
788+
'sbe.target.namespace': 'gen.car_example')
789+
args = ['sbe-samples/src/main/resources/example-schema.xml']
790+
}
791+
792+
task generatePythonIssue435Codecs(type: JavaExec) {
793+
main = 'uk.co.real_logic.sbe.SbeTool'
794+
classpath = project(':sbe-all').sourceSets.main.runtimeClasspath
795+
systemProperties(
796+
'sbe.output.dir': 'python/tests',
797+
'sbe.target.language': 'uk.co.real_logic.sbe.generation.python.Python',
798+
'sbe.target.namespace': 'gen.issue435')
799+
args = ['sbe-tool/src/test/resources/issue435.xml']
800+
}
801+
802+
task generatePythonIssue483Codecs(type: JavaExec) {
803+
main = 'uk.co.real_logic.sbe.SbeTool'
804+
classpath = project(':sbe-all').sourceSets.main.runtimeClasspath
805+
systemProperties(
806+
'sbe.output.dir': 'python/tests',
807+
'sbe.target.language': 'uk.co.real_logic.sbe.generation.python.Python',
808+
'sbe.target.namespace': 'gen.issue483')
809+
args = ['sbe-tool/src/test/resources/issue483.xml']
810+
}
811+
812+
task generatePythonIssue560Codecs(type: JavaExec) {
813+
main = 'uk.co.real_logic.sbe.SbeTool'
814+
classpath = project(':sbe-all').sourceSets.main.runtimeClasspath
815+
systemProperties(
816+
'sbe.output.dir': 'python/tests',
817+
'sbe.target.language': 'uk.co.real_logic.sbe.generation.python.Python',
818+
'sbe.target.namespace': 'gen.issue560')
819+
args = ['sbe-tool/src/test/resources/issue560.xml']
820+
}
821+
822+
task generatePythonFixBinaryCodecs(type: JavaExec) {
823+
main = 'uk.co.real_logic.sbe.SbeTool'
824+
classpath = project(':sbe-all').sourceSets.main.runtimeClasspath
825+
systemProperties(
826+
'sbe.output.dir': 'python/tests',
827+
'sbe.target.language': 'uk.co.real_logic.sbe.generation.python.Python',
828+
'sbe.target.namespace': 'gen.fix_binary')
829+
args = [
830+
'sbe-tool/src/test/resources/FixBinary.xml',
831+
]
832+
}
833+
834+
task generatePythonExampleDataFile(type: JavaExec) {
835+
mkdir "python/tests/gen/car_example"
836+
main = 'uk.co.real_logic.sbe.examples.ExampleUsingGeneratedStub'
837+
classpath = project(':sbe-samples').sourceSets.main.runtimeClasspath
838+
systemProperties('sbe.encoding.filename': 'python/tests/gen/car_example/car_example_data.sbe')
839+
args = []
840+
}
841+
842+
task generatePythonCodecs {
843+
description = 'Generate python test codecs'
844+
delete 'python/tests/gen'
845+
dependsOn 'generatePythonCarExample', 'generatePythonExampleDataFile', 'generatePythonIssue435Codecs',
846+
'generatePythonIssue483Codecs', 'generatePythonIssue560Codecs', 'generatePythonFixBinaryCodecs'
847+
}
848+
849+
task runPythonTests(type: Exec) {
850+
workingDir = './python/'
851+
executable = 'python'
852+
args = ['-m','unittest','discover']
853+
dependsOn 'generatePythonCodecs'
854+
}
855+
778856
wrapper {
779857
gradleVersion = '5.6.2'
780858
distributionType = 'ALL'

‎python/tests/__init__.py‎

Whitespace-only changes.

‎python/tests/test_car_example.py‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import unittest
2+
from pathlib import Path
3+
4+
from tests.gen.car_example import MessageHeaderEncoder, CarEncoder, BooleanType, Model, EngineEncoder, BoostType
5+
from tests.gen.car_example import MessageHeaderDecoder, CarDecoder, EngineDecoder
6+
7+
8+
class TestCarExample(unittest.TestCase):
9+
REFERENCE_FILE_ENCODE = Path(__file__).resolve().parent / Path('gen') / 'car_example' / 'car_example_data.sbe'
10+
11+
VEHICLE_CODE = "abcdef".encode(CarDecoder.vehicleCodeCharacterEncoding())
12+
MANUFACTURER_CODE = "123".encode(EngineDecoder.manufacturerCodeCharacterEncoding())
13+
MANUFACTURER = "Honda".encode(CarDecoder.manufacturerCharacterEncoding())
14+
MODEL = "Civic VTi".encode(CarDecoder.modelCharacterEncoding())
15+
ACTIVATION_CODE = "abcdef".encode(CarDecoder.activationCodeCharacterEncoding())
16+
17+
CAR_ENCODER = CarEncoder()
18+
MESSAGE_HEADER_ENCODER = MessageHeaderEncoder()
19+
20+
CAR_DECODER = CarDecoder()
21+
MESSAGE_HEADER_DECODER = MessageHeaderDecoder()
22+
23+
def test_encoding(self):
24+
with open(self.REFERENCE_FILE_ENCODE, 'rb') as f:
25+
buffer = f.read()
26+
py_buf = bytearray(4096)
27+
self._encode(py_buf, self.MESSAGE_HEADER_ENCODER, self.CAR_ENCODER)
28+
# Encoded message should match
29+
self.assertEqual(buffer, py_buf[:207])
30+
# The empty part of the buffer should be all 0's
31+
self.assertEqual(bytes(len(py_buf) - len(buffer)), py_buf[len(buffer):])
32+
self.assertEqual(207, len(buffer))
33+
34+
def test_decoding(self):
35+
self.maxDiff = None
36+
with open(self.REFERENCE_FILE_ENCODE, 'rb') as f:
37+
buffer = f.read()
38+
py_buf = bytearray(4096)
39+
self._encode(py_buf, self.MESSAGE_HEADER_ENCODER, self.CAR_ENCODER)
40+
self._decode(py_buf, self.MESSAGE_HEADER_DECODER, self.CAR_DECODER)
41+
self._decode(buffer, self.MESSAGE_HEADER_DECODER, self.CAR_DECODER)
42+
43+
def _encode(self, buf: bytearray, header: MessageHeaderEncoder, car: CarEncoder):
44+
offset = 0
45+
46+
car.wrapAndApplyHeader(buf, offset, header)
47+
car.serialNumber(1234)
48+
car.modelYear(2013)
49+
car.available(BooleanType.T)
50+
car.code(Model.A)
51+
car.set_vehicleCode(self.VEHICLE_CODE)
52+
53+
car.someNumbers(0,1)
54+
car.someNumbers(1,2)
55+
car.someNumbers(2,3)
56+
car.someNumbers(3,4)
57+
58+
car.extras() \
59+
.clear() \
60+
.cruiseControl(True) \
61+
.sportsPack(True) \
62+
.sunRoof(False)
63+
64+
car.engine() \
65+
.capacity(2000) \
66+
.numCylinders(4) \
67+
.set_manufacturerCode(self.MANUFACTURER_CODE) \
68+
.efficiency(35) \
69+
.boosterEnabled(BooleanType.T) \
70+
.booster() \
71+
.boostType(BoostType.NITROUS) \
72+
.horsePower(200)
73+
74+
ff = car.fuelFiguresCount(3)
75+
ff.next().speed(30).mpg(35.9).usageDescription("Urban Cycle".encode(CarEncoder.FuelFiguresEncoder
76+
.usageDescriptionCharacterEncoding()))
77+
ff.next().speed(55).mpg(49.0).usageDescription("Combined Cycle".encode(CarEncoder.FuelFiguresEncoder
78+
.usageDescriptionCharacterEncoding()))
79+
ff.next().speed(75).mpg(40.0).usageDescription("Highway Cycle".encode(CarEncoder.FuelFiguresEncoder
80+
.usageDescriptionCharacterEncoding()))
81+
82+
figures = car.performanceFiguresCount(2)
83+
figures.next().octaneRating(95).accelerationCount(3) \
84+
.next().mph(30).seconds(4.0) \
85+
.next().mph(60).seconds(7.5) \
86+
.next().mph(100).seconds(12.2)
87+
88+
figures.next().octaneRating(99).accelerationCount(3) \
89+
.next().mph(30).seconds(3.8) \
90+
.next().mph(60).seconds(7.1) \
91+
.next().mph(100).seconds(11.8)
92+
93+
car.manufacturer(self.MANUFACTURER).model(self.MODEL).activationCode(self.ACTIVATION_CODE)
94+
95+
return header.encodedLength + car.encodedLength
96+
97+
def _decode(self, buf: bytes, header: MessageHeaderDecoder, car: CarDecoder):
98+
offset = 0
99+
out = ""
100+
header.wrap(buf, offset)
101+
template_id = header.templateId()
102+
if template_id != car.sbeTemplateId:
103+
raise Exception("Template ID missmatch")
104+
acting_block_length = header.blockLength()
105+
acting_block_version = header.version()
106+
offset += header.encodedLength
107+
108+
car.wrap(buf, offset, acting_block_length, acting_block_version)
109+
self.assertEqual(1234, car.serialNumber())
110+
self.assertEqual(2013, car.modelYear())
111+
self.assertEqual(BooleanType.T, car.available())
112+
self.assertEqual(Model.A, car.code())
113+
self.assertEqual((1, 2, 3, 4), car.getMultiSomeNumbers())
114+
self.assertEqual(1, car.someNumbers(0))
115+
self.assertEqual(2, car.someNumbers(1))
116+
self.assertEqual(3, car.someNumbers(2))
117+
self.assertEqual(4, car.someNumbers(3))
118+
index_oob = False
119+
try:
120+
car.someNumbers(55)
121+
except IndexError as e:
122+
index_oob = True
123+
self.assertEqual(IndexError, type(e))
124+
self.assertTrue(index_oob)
125+
126+
self.assertEqual("abcdef", str(car.getMultiVehicleCode(), car.vehicleCodeCharacterEncoding()))
127+
128+
extras = car.extras()
129+
self.assertTrue(extras.cruiseControl())
130+
self.assertTrue(extras.sportsPack())
131+
self.assertFalse(extras.sunRoof())
132+
133+
self.assertEqual("C", car.discountedModel().name)
134+
135+
engine = car.engine()
136+
self.assertEqual(2000, engine.capacity())
137+
self.assertEqual(4, engine.numCylinders())
138+
self.assertEqual(9000, engine.maxRpm)
139+
self.assertEqual("123", str(engine.getMultiManufacturerCode(), EngineDecoder.manufacturerCodeCharacterEncoding()))
140+
self.assertEqual(35, engine.efficiency())
141+
self.assertEqual(BooleanType.T, engine.boosterEnabled())
142+
self.assertEqual(BoostType.NITROUS, engine.booster().boostType())
143+
self.assertEqual(200, engine.booster().horsePower())
144+
145+
self.assertEqual("Petrol", str(engine.getFuel(0, len(buf)).tobytes().decode()))
146+
147+
ff = car.fuelFigures()
148+
ff.next()
149+
self.assertEqual(30, ff.speed())
150+
self.assertAlmostEqual(35.9, ff.mpg(), places=5)
151+
self.assertEqual("Urban Cycle", ff.usageDescription())
152+
ff.next()
153+
self.assertEqual(55, ff.speed())
154+
self.assertAlmostEqual(49.0, ff.mpg(), places=5)
155+
self.assertEqual("Combined Cycle", ff.usageDescription())
156+
ff.next()
157+
self.assertEqual(75, ff.speed())
158+
self.assertAlmostEqual(40.0, ff.mpg(), places=5)
159+
self.assertEqual("Highway Cycle", ff.usageDescription())
160+
161+
pf = car.performanceFigures()
162+
pf.next()
163+
self.assertEqual(95, pf.octaneRating())
164+
pfa = pf.acceleration()
165+
pfa.next()
166+
self.assertEqual(30, pfa.mph())
167+
self.assertAlmostEqual(4.0, pfa.seconds(), places=5)
168+
pfa.next()
169+
self.assertEqual(60, pfa.mph())
170+
self.assertAlmostEqual(7.5, pfa.seconds(), places=5)
171+
pfa.next()
172+
self.assertEqual(100, pfa.mph())
173+
self.assertAlmostEqual(12.2, pfa.seconds(), places=5)
174+
175+
pf.next()
176+
self.assertEqual(99, pf.octaneRating())
177+
pfa = pf.acceleration()
178+
pfa.next()
179+
self.assertEqual(30, pfa.mph())
180+
self.assertAlmostEqual(3.8, pfa.seconds(), places=5)
181+
pfa.next()
182+
self.assertEqual(60, pfa.mph())
183+
self.assertAlmostEqual(7.1, pfa.seconds(), places=5)
184+
pfa.next()
185+
self.assertEqual(100, pfa.mph())
186+
self.assertAlmostEqual(11.8, pfa.seconds(), places=5)
187+
188+
self.assertEqual("Honda", car.manufacturer())
189+
self.assertEqual("Civic VTi", car.model())
190+
self.assertEqual("abcdef", car.activationCode())
191+
self.assertEqual(199, car.encodedLength)
192+
return out

‎python/tests/test_issue435.py‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import unittest
2+
3+
from tests.gen.issue435.message_header_encoder import *
4+
from tests.gen.issue435.message_header_decoder import *
5+
from tests.gen.issue435.issue435_encoder import *
6+
from tests.gen.issue435.issue435_decoder import *
7+
8+
9+
class TestIssue435(unittest.TestCase):
10+
11+
def setUp(self) -> None:
12+
self._buffer = bytearray(4096)
13+
self._messageHeader = MessageHeaderEncoder()
14+
self._messageHeaderDecoder = MessageHeaderDecoder()
15+
self._issue435 = Issue435Encoder()
16+
self._issue435_decoder = Issue435Decoder()
17+
self._messageHeader.wrap(self._buffer, 0)
18+
self._messageHeader.version(self._issue435.sbeSchemaVersion)
19+
self._messageHeader.blockLength(self._issue435.sbeBlockLength)
20+
self._messageHeader.schemaId(self._issue435.sbeSchemaId)
21+
self._messageHeader.templateId(self._issue435.sbeTemplateId)
22+
23+
#< ref > element in non - standard
24+
set_s = self._messageHeader.s()
25+
set_s.one(True)
26+
27+
def test_non_standard_header_size(self):
28+
self.assertEqual(9, self._messageHeader.encodedLength)
29+
30+
def test_issue435_ref_test(self):
31+
self._issue435.wrap(self._buffer, self._messageHeader.encodedLength)
32+
ex = self._issue435.example()
33+
ex.e(EnumRef.Two)
34+
self.assertEqual(1, self._issue435.sbeBlockLength)
35+
self._messageHeaderDecoder.wrap(self._buffer, 0)
36+
self._messageHeader.wrap(self._buffer, 0)
37+
self.assertEqual(self._issue435.sbeBlockLength, self._messageHeaderDecoder.blockLength(), "Incorrect BlockLength")
38+
self.assertEqual(self._issue435.sbeSchemaId, self._messageHeaderDecoder.schemaId(), "Incorrect SchemaId")
39+
self.assertEqual(self._issue435.sbeTemplateId, self._messageHeaderDecoder.templateId(), "Incorrect TemplateId")
40+
self.assertEqual(self._issue435.sbeSchemaVersion, self._messageHeaderDecoder.version(), "Incorrect SchemaVersion")
41+
self.assertTrue(self._messageHeaderDecoder.s().one(), "Incorrect SetRef.One")
42+
43+
self._issue435_decoder.wrap(self._buffer, self._messageHeader.encodedLength, self._messageHeaderDecoder.blockLength(),
44+
self._messageHeaderDecoder.version())
45+
46+
self.assertEqual(EnumRef.Two, self._issue435_decoder.example().e(), "Incorrect EnuRef");

‎python/tests/test_issue483.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import unittest
2+
3+
from tests.gen.issue483.issue483_encoder import *
4+
from tests.gen.issue483.issue483_decoder import *
5+
6+
7+
class TestIssue483(unittest.TestCase):
8+
9+
def test_presense(self):
10+
# Check attributes for their presence meta attribute
11+
self.assertEqual(Issue483Encoder.unsetMetaAttribute(MetaAttribute.Presence), "required")
12+
self.assertEqual(Issue483Encoder.requiredMetaAttribute(MetaAttribute.Presence), "required")
13+
self.assertEqual(Issue483Encoder.constantMetaAttribute(MetaAttribute.Presence), "constant")
14+
self.assertEqual(Issue483Encoder.optionalMetaAttribute(MetaAttribute.Presence), "optional")

‎python/tests/test_issue560.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import unittest
2+
3+
from tests.gen.issue560 import Model
4+
from tests.gen.issue560.issue560_encoder import Issue560Encoder
5+
from tests.gen.issue560.issue560_decoder import Issue560Decoder
6+
from tests.gen.issue560.message_header_encoder import MessageHeaderEncoder
7+
8+
9+
class TestIssue560(unittest.TestCase):
10+
11+
def setUp(self) -> None:
12+
self._buffer = bytearray(4096)
13+
self._issue560 = Issue560Encoder()
14+
self._issue560_decoder = Issue560Decoder()
15+
self._header = MessageHeaderEncoder()
16+
self._header.wrap(self._buffer, 0)
17+
self._header.blockLength(self._issue560.BLOCK_LENGTH)
18+
self._header.schemaId(self._issue560.SCHEMA_ID)
19+
self._header.templateId(self._issue560.TEMPLATE_ID)
20+
self._header.version(self._issue560.SCHEMA_VERSION)
21+
22+
def test_const_enum(self):
23+
self.assertEqual(self._issue560_decoder.discountedModel(), Model.C, "Incorrect const enum valueref")

0 commit comments

Comments
 (0)