edi fileformat - all about segment/segment group

extensions/omniv21/fileformat/edi/seg.go

@@ -0,0 +1,127 @@
+package edi
+
+import (
+"github.com/jf-tech/go-corelib/maths"
+"github.com/jf-tech/go-corelib/strs"
+)
+
+const (
+segTypeSeg = "segment"
+segTypeGroup = "segment_group"
+)
+
+const (
+fqdnDelim = "/"
+)
+
+type elem struct {
+Name string `json:"name,omitempty"`
+Index int `json:"index,omitempty"`
+ComponentIndex *int `json:"component_index,omitempty"`
+EmptyIfMissing bool `json:"empty_if_missing,omitempty"`
+}
+
+type segDecl struct {
+Name string `json:"name,omitempty"`
+Type *string `json:"type,omitempty"`
+IsTarget bool `json:"is_target,omitempty"`
+Min *int `json:"min,omitempty"`
+Max *int `json:"max,omitempty"`
+Elems []elem `json:"elements,omitempty"`
+Children []*segDecl `json:"child_segments,omitempty"`
+}
+
+func (d *segDecl) isGroup() bool {
+return d.Type != nil && *d.Type == segTypeGroup
+}
+
+func (d *segDecl) minOccurs() int {
+switch d.Min {
+case nil:
+return 1
+default:
+return *d.Min
+}
+}
+
+func (d *segDecl) maxOccurs() int {
+switch {
+case d.Max == nil:
+return 1
+case *d.Max < 0:
+return maths.MaxIntValue
+default:
+return *d.Max
+}
+}
+
+type segDeclRuntime struct {
+decl *segDecl
+children []*segDeclRuntime
+parent *segDeclRuntime
+containsTarget bool // whether this seg itself or any of its children has is_target == true
+occurred int
+fqdn string
+}
+
+const (
+rootSegName = "#root"
+)
+
+func newSegDeclRuntimeTree(segDecls ...*segDecl) *segDeclRuntime {
+return newSegDeclRuntime(
+nil,
+&segDecl{
+Name: rootSegName,
+Type: strs.StrPtr(segTypeGroup),
+Children: segDecls,
+})
+}
+
+func newSegDeclRuntime(parent *segDeclRuntime, segDecl *segDecl) *segDeclRuntime {
+rt := segDeclRuntime{decl: segDecl, parent: parent}
+if parent != nil {
+// we don't set fqdn on root.
+if parent.fqdn != "" {
+rt.fqdn = strs.BuildFQDN2(fqdnDelim, parent.fqdn, segDecl.Name)
+} else {
+// we don't include rootSegName as part of fqdn.
+rt.fqdn = segDecl.Name
+}
+}
+containsTarget := segDecl.IsTarget
+for _, child := range segDecl.Children {
+childRt := newSegDeclRuntime(&rt, child)
+containsTarget = containsTarget || childRt.containsTarget
+rt.children = append(rt.children, childRt)
+}
+rt.containsTarget = containsTarget
+return &rt
+}
+
+func (rt *segDeclRuntime) matchSegName(segName string) bool {
+switch rt.decl.isGroup() {
+case true:
+// Group (or so called loop) itself doesn't have a segment name in EDI file (we do assign a
+// name to it for xpath query reference, but that name isn't a segment name per se). A
+// group/loop's first non-group child, recursively if necessary, can be used as the group's
+// identifying segment name, per EDI standard. Meaning if a group's first non-group child's
+// segment exists in an EDI file, then this group has an instance in the file. If the first
+// non-group child's segment isn't found, then we (the standard) assume the group is skipped.
+// The explanation can be found:
+// - https://www.gxs.co.uk/wp-content/uploads/tutorial_ansi.pdf (around page 9), quote
+// "...loop is optional, but if any segment in the loop is used, the first segment
+// within the loop becomes mandatory..."
+// - https://github.com/smooks/smooks-edi-cartridge/blob/54f97e89156114e13e1acd3b3c46fe9a4234918c/edi-sax/src/main/java/org/smooks/edi/edisax/model/internal/SegmentGroup.java#L68
+return len(rt.children) > 0 && rt.children[0].matchSegName(segName)
+default:
+return rt.decl.Name == segName
+}
+}
+
+func (rt *segDeclRuntime) resetChildrenOccurred() {
+for _, child := range rt.children {
+child.occurred = 0
+child.resetChildrenOccurred()
+}
+}

extensions/omniv21/fileformat/edi/seg_test.go

@@ -0,0 +1,209 @@
+package edi
+
+import (
+"testing"
+
+"github.com/jf-tech/go-corelib/maths"
+"github.com/jf-tech/go-corelib/strs"
+"github.com/jf-tech/go-corelib/testlib"
+"github.com/stretchr/testify/assert"
+)
+
+func TestSegDeclIsGroup(t *testing.T) {
+assert.False(t, (&segDecl{}).isGroup())
+assert.False(t, (&segDecl{Type: strs.StrPtr(segTypeSeg)}).isGroup())
+assert.False(t, (&segDecl{Type: strs.StrPtr("something")}).isGroup())
+assert.True(t, (&segDecl{Type: strs.StrPtr(segTypeGroup)}).isGroup())
+}
+
+func TestSegDeclMinMaxOccurs(t *testing.T) {
+for _, test := range []struct {
+name string
+min *int
+max *int
+expectedMin int
+expectedMax int
+}{
+{
+name: "default",
+min: nil,
+max: nil,
+expectedMin: 1,
+expectedMax: 1,
+},
+{
+name: "min/max=0",
+min: testlib.IntPtr(0),
+max: testlib.IntPtr(0),
+expectedMin: 0,
+expectedMax: 0,
+},
+{
+name: "max unlimited",
+min: nil,
+max: testlib.IntPtr(-1),
+expectedMin: 1,
+expectedMax: maths.MaxIntValue,
+},
+{
+name: "min/max finite",
+min: testlib.IntPtr(3),
+max: testlib.IntPtr(5),
+expectedMin: 3,
+expectedMax: 5,
+},
+} {
+t.Run(test.name, func(t *testing.T) {
+s := &segDecl{Min: test.min, Max: test.max}
+assert.Equal(t, test.expectedMin, s.minOccurs())
+assert.Equal(t, test.expectedMax, s.maxOccurs())
+})
+}
+}
+
+func verifySegDeclRt(t *testing.T, rt *segDeclRuntime, decl *segDecl, parentRt *segDeclRuntime) {
+assert.NotNil(t, rt)
+assert.Equal(t, len(decl.Children), len(rt.children))
+assert.True(t, rt.parent == parentRt)
+assert.Equal(t, 0, rt.occurred)
+assert.Equal(t, *decl, *rt.decl)
+}
+
+func verifyRoot(t *testing.T, root *segDeclRuntime, childSegDecls ...*segDecl) {
+verifySegDeclRt(
+t,
+root,
+&segDecl{
+Name: rootSegName,
+Type: strs.StrPtr(segTypeGroup),
+Children: childSegDecls,
+},
+nil)
+}
+
+func TestNewSegDeclRuntimeTree_Empty(t *testing.T) {
+verifyRoot(t, newSegDeclRuntimeTree())
+}
+
+func TestNewSegDeclRuntimeTree(t *testing.T) {
+/*
+ ISA
+ GS
+ ST
+ B10
+ GE
+ IEA
+*/
+st := &segDecl{Name: "ST"}
+b10 := &segDecl{Name: "B10"}
+gs := &segDecl{Name: "GS", Children: []*segDecl{st, b10}}
+ge := &segDecl{Name: "GE"}
+isa := &segDecl{Name: "ISA", Children: []*segDecl{gs, ge}}
+iea := &segDecl{Name: "IEA"}
+
+r := newSegDeclRuntimeTree(isa, iea)
+verifyRoot(t, r, isa, iea)
+
+isaRt := r.children[0]
+verifySegDeclRt(t, isaRt, isa, r)
+assert.Equal(t, "ISA", isaRt.fqdn)
+
+ieaRt := r.children[1]
+verifySegDeclRt(t, ieaRt, iea, r)
+assert.Equal(t, "IEA", ieaRt.fqdn)
+
+gsRt := isaRt.children[0]
+verifySegDeclRt(t, gsRt, gs, isaRt)
+assert.Equal(t, "ISA/GS", gsRt.fqdn)
+
+geRt := isaRt.children[1]
+verifySegDeclRt(t, geRt, ge, isaRt)
+assert.Equal(t, "ISA/GE", geRt.fqdn)
+
+stRt := gsRt.children[0]
+verifySegDeclRt(t, stRt, st, gsRt)
+assert.Equal(t, "ISA/GS/ST", stRt.fqdn)
+
+b10Rt := gsRt.children[1]
+verifySegDeclRt(t, b10Rt, b10, gsRt)
+assert.Equal(t, "ISA/GS/B10", b10Rt.fqdn)
+}
+
+func TestMatchSegName(t *testing.T) {
+/*
+ ISA
+ GS
+ ST
+ B10
+ GE
+*/
+b10 := &segDecl{Name: "B10"}
+st := &segDecl{Name: "ST", Type: strs.StrPtr(segTypeGroup), Children: []*segDecl{b10}}
+gs := &segDecl{Name: "GS", Type: strs.StrPtr(segTypeGroup), Children: []*segDecl{st}}
+ge := &segDecl{Name: "GE"}
+isa := &segDecl{Name: "ISA", Children: []*segDecl{gs, ge}}
+
+r := newSegDeclRuntimeTree(isa)
+verifyRoot(t, r, isa)
+
+isaRt := r.children[0]
+assert.True(t, isaRt.matchSegName("ISA"))
+assert.False(t, isaRt.matchSegName("GS"))
+assert.False(t, isaRt.matchSegName("ST"))
+assert.False(t, isaRt.matchSegName("B10"))
+
+gsRt := isaRt.children[0]
+assert.False(t, gsRt.matchSegName("GS"))
+assert.False(t, gsRt.matchSegName("ST"))
+assert.True(t, gsRt.matchSegName("B10"))
+
+stRt := gsRt.children[0]
+assert.False(t, stRt.matchSegName("ST"))
+assert.True(t, stRt.matchSegName("B10"))
+}
+
+func TestResetChildrenOccurred(t *testing.T) {
+/*
+ ISA
+ GS
+ ST
+ B10
+ GE
+ IEA
+*/
+st := &segDecl{Name: "ST"}
+b10 := &segDecl{Name: "B10"}
+gs := &segDecl{Name: "GS", Children: []*segDecl{st, b10}}
+ge := &segDecl{Name: "GE"}
+isa := &segDecl{Name: "ISA", Children: []*segDecl{gs, ge}}
+iea := &segDecl{Name: "IEA"}
+
+r := newSegDeclRuntimeTree(isa, iea)
+verifyRoot(t, r, isa, iea)
+
+isaRt := r.children[0]
+isaRt.occurred = 1
+
+ieaRt := r.children[1]
+ieaRt.occurred = 2
+
+gsRt := isaRt.children[0]
+gsRt.occurred = 3
+
+geRt := isaRt.children[1]
+geRt.occurred = 4
+
+stRt := gsRt.children[0]
+stRt.occurred = 5
+
+b10Rt := gsRt.children[1]
+b10Rt.occurred = 6
+
+gsRt.resetChildrenOccurred()
+assert.Equal(t, 0, stRt.occurred)
+assert.Equal(t, 0, b10Rt.occurred)
+assert.Equal(t, 1, isaRt.occurred)
+assert.Equal(t, 2, ieaRt.occurred)
+assert.Equal(t, 3, gsRt.occurred)
+assert.Equal(t, 4, geRt.occurred)
+}

extensions/omniv21/samples/csv/1_weather_data_csv.schema.json

@@ -35,8 +35,8 @@
  "name": "dateTimeToRFC3339",
  "args": [
  { "xpath": "DATE" },
- { "const": "", "keep_empty_or_null": true, "_comment": "input timezone" },
- { "const": "", "keep_empty_or_null": true, "_comment": "output timezone" }
+ { "const": "", "_comment": "input timezone" },
+ { "const": "", "_comment": "output timezone" }
  ]
  }},
  "high_temperature_fahrenheit": { "xpath": "HIGH_TEMP_C", "template": "template_c_to_f" },