fixed-length fileformat schema validation (#112)

fixed-length fileformat schema validation

Author: jf-tech

extensions/omniv21/fileformat/csv/format_test.go

@@ -156,7 +156,7 @@ func TestValidateSchema(t *testing.T) {
 }
 
 func TestCreateFormatReader(t *testing.T) {
-r, err := NewCSVFileFormat("test-schema").CreateFormatReader(
+r, err := NewCSVFileFormat("test").CreateFormatReader(
 "test-input",
 strings.NewReader(
 lf("A|B|C")+

extensions/omniv21/fileformat/fixedlength/.snapshots/TestValidateSchema-success_-_by_header_footer

@@ -0,0 +1,44 @@
+{
+"file_declaration": {
+"envelopes": [
+{
+"name": "1",
+"by_header_footer": {
+"header": "^FILE-BEGIN$",
+"footer": "^FILE-BEGIN$"
+},
+"by_rows": null,
+"not_target": true,
+"columns": null
+},
+{
+"name": "2",
+"by_header_footer": {
+"header": "^DATA-BLOCK-BEGIN$",
+"footer": "^DATA-BLOCK-END$"
+},
+"by_rows": null,
+"not_target": false,
+"columns": [
+{
+"name": "abc",
+"start_pos": 1,
+"length": 3,
+"line_pattern": "^DATA:.*$"
+}
+]
+},
+{
+"name": "3",
+"by_header_footer": {
+"header": "^FILE-END$",
+"footer": "^FILE-END$"
+},
+"by_rows": null,
+"not_target": true,
+"columns": null
+}
+]
+},
+"XPath": ""
+}

extensions/omniv21/fileformat/fixedlength/.snapshots/TestValidateSchema-success_-_by_rows=3

@@ -0,0 +1,27 @@
+{
+"file_declaration": {
+"envelopes": [
+{
+"name": "1",
+"by_header_footer": null,
+"by_rows": 3,
+"not_target": false,
+"columns": [
+{
+"name": "abc",
+"start_pos": 1,
+"length": 10,
+"line_pattern": "^L01.*"
+},
+{
+"name": "efg",
+"start_pos": 3,
+"length": 5,
+"line_pattern": "^L03.*"
+}
+]
+}
+]
+},
+"XPath": ""
+}

extensions/omniv21/fileformat/fixedlength/.snapshots/TestValidateSchema-success_-_simple_by_rows=1

@@ -0,0 +1,21 @@
+{
+"file_declaration": {
+"envelopes": [
+{
+"name": "1",
+"by_header_footer": null,
+"by_rows": null,
+"not_target": false,
+"columns": [
+{
+"name": "abc",
+"start_pos": 1,
+"length": 10,
+"line_pattern": null
+}
+]
+}
+]
+},
+"XPath": ".[abc != 'skip']"
+}

extensions/omniv21/fileformat/fixedlength/decl.go

@@ -12,30 +12,18 @@ type byHeaderFooterDecl struct {
 }
 
 type columnDecl struct {
-Name string `json:"name"`
-StartPos int `json:"start_pos"` // 1-based. and rune-based.
-Length int `json:"length"` // rune-based length.
-Line *string `json:"line"`
-}
-
-type envelopeDecl struct {
-Name *string `json:"name"`
-ByHeaderFooter *byHeaderFooterDecl `json:"by_header_footer"`
-ByRows *int `json:"by_rows"`
-NotTarget bool `json:"not_target"`
-Columns []*columnDecl `json:"columns"`
-}
-
-type fileDecl struct {
-Envelopes []*envelopeDecl `json:"envelopes"`
+Name string `json:"name"`
+StartPos int `json:"start_pos"` // 1-based. and rune-based.
+Length int `json:"length"` // rune-based length.
+LinePattern *string `json:"line_pattern"`
 }
 
 func (c *columnDecl) lineMatch(line []byte) bool {
-if c.Line == nil {
+if c.LinePattern == nil {
 return true
 }
 // validated in validation code.
-r, _ := caches.GetRegex(*c.Line)
+r, _ := caches.GetRegex(*c.LinePattern)
 return r.Match(line)
 }
 
@@ -53,6 +41,14 @@ func (c *columnDecl) lineToColumn(line []rune) []rune {
 return nil
 }
 
+type envelopeDecl struct {
+Name *string `json:"name"`
+ByHeaderFooter *byHeaderFooterDecl `json:"by_header_footer"`
+ByRows *int `json:"by_rows"`
+NotTarget bool `json:"not_target"`
+Columns []*columnDecl `json:"columns"`
+}
+
 func (e *envelopeDecl) byRows() int {
 if e.ByHeaderFooter != nil {
 panic(fmt.Sprintf("envelope '%s' type is not 'by_rows'", *e.Name))
@@ -62,3 +58,21 @@ func (e *envelopeDecl) byRows() int {
 }
 return *e.ByRows
 }
+
+type fileDecl struct {
+Envelopes []*envelopeDecl `json:"envelopes"`
+}
+
+type envelopeType int
+
+const (
+envelopeTypeByRows envelopeType = iota
+envelopeTypeByHeaderFooter
+)
+
+func (f *fileDecl) envelopeType() envelopeType {
+if f.Envelopes[0].ByHeaderFooter != nil {
+return envelopeTypeByHeaderFooter
+}
+return envelopeTypeByRows
+}

extensions/omniv21/fileformat/fixedlength/decl_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestColumnDecl_LineMatch(t *testing.T) {
 assert.True(t, (&columnDecl{}).lineMatch([]byte("test")))
-assert.False(t, (&columnDecl{Line: strs.StrPtr("^ABC.*$")}).lineMatch([]byte("test")))
-assert.True(t, (&columnDecl{Line: strs.StrPtr("^ABC.*$")}).lineMatch([]byte("ABCDEFG")))
+assert.False(t, (&columnDecl{LinePattern: strs.StrPtr("^ABC.*$")}).lineMatch([]byte("test")))
+assert.True(t, (&columnDecl{LinePattern: strs.StrPtr("^ABC.*$")}).lineMatch([]byte("ABCDEFG")))
 }
 
 func TestColumnDecl_LineToColumn(t *testing.T) {
@@ -30,3 +30,24 @@ func TestEnvelopeDecl_ByRows(t *testing.T) {
 assert.Equal(t, 1, (&envelopeDecl{}).byRows())
 assert.Equal(t, 12, (&envelopeDecl{ByRows: testlib.IntPtr(12)}).byRows())
 }
+
+func TestFileDecl_EnvelopeType(t *testing.T) {
+assert.Equal(t, envelopeTypeByHeaderFooter,
+(&fileDecl{
+Envelopes: []*envelopeDecl{
+{ByHeaderFooter: &byHeaderFooterDecl{}},
+},
+}).envelopeType())
+assert.Equal(t, envelopeTypeByRows,
+(&fileDecl{
+Envelopes: []*envelopeDecl{
+{ByRows: testlib.IntPtr(12)},
+},
+}).envelopeType())
+assert.Equal(t, envelopeTypeByRows,
+(&fileDecl{
+Envelopes: []*envelopeDecl{
+{},
+},
+}).envelopeType())
+}

extensions/omniv21/fileformat/fixedlength/format.go

@@ -0,0 +1,139 @@
+package fixedlength
+
+import (
+"encoding/json"
+"fmt"
+"io"
+"strconv"
+"strings"
+
+"github.com/jf-tech/go-corelib/caches"
+"github.com/jf-tech/go-corelib/strs"
+
+"github.com/jf-tech/omniparser/errs"
+"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
+"github.com/jf-tech/omniparser/extensions/omniv21/transform"
+v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
+"github.com/jf-tech/omniparser/validation"
+)
+
+const (
+fileFormatFixedLength = "fixed-length"
+)
+
+type fixedLengthFileFormat struct {
+schemaName string
+autoGenEnvelopeNameIndex int
+}
+
+// NewFixedLengthFileFormat creates a FileFormat for fixed-length files.
+func NewFixedLengthFileFormat(schemaName string) fileformat.FileFormat {
+return &fixedLengthFileFormat{schemaName: schemaName}
+}
+
+type fixedLengthFormatRuntime struct {
+Decl *fileDecl `json:"file_declaration"`
+XPath string
+}
+
+func (f *fixedLengthFileFormat) ValidateSchema(
+format string, schemaContent []byte, finalOutputDecl *transform.Decl) (interface{}, error) {
+if format != fileFormatFixedLength {
+return nil, errs.ErrSchemaNotSupported
+}
+err := validation.SchemaValidate(f.schemaName, schemaContent, v21validation.JSONSchemaFixedLengthFileDeclaration)
+if err != nil {
+// err is already context formatted.
+return nil, err
+}
+var runtime fixedLengthFormatRuntime
+_ = json.Unmarshal(schemaContent, &runtime) // JSON schema validation earlier guarantees Unmarshal success.
+err = f.validateFileDecl(runtime.Decl)
+if err != nil {
+// err is already context formatted.
+return nil, err
+}
+if finalOutputDecl == nil {
+return nil, f.FmtErr("'FINAL_OUTPUT' is missing")
+}
+runtime.XPath = strings.TrimSpace(strs.StrPtrOrElse(finalOutputDecl.XPath, ""))
+if runtime.XPath != "" {
+_, err := caches.GetXPathExpr(runtime.XPath)
+if err != nil {
+return nil, f.FmtErr("'FINAL_OUTPUT.xpath' (value: '%s') is invalid, err: %s",
+runtime.XPath, err.Error())
+}
+}
+return &runtime, nil
+}
+
+func (f *fixedLengthFileFormat) validateFileDecl(decl *fileDecl) error {
+targetSeen := false
+namesSeen := map[string]bool{}
+for _, envelope := range decl.Envelopes {
+if targetSeen && !envelope.NotTarget {
+return f.FmtErr("cannot have more than one target envelope")
+}
+targetSeen = targetSeen || !envelope.NotTarget
+if envelope.Name == nil {
+f.autoGenEnvelopeNameIndex++
+envelope.Name = strs.StrPtr(strconv.Itoa(f.autoGenEnvelopeNameIndex))
+}
+if _, found := namesSeen[*envelope.Name]; found {
+return f.FmtErr("more than one envelope has the name '%s'", *envelope.Name)
+}
+namesSeen[*envelope.Name] = true
+if err := f.validateByHeaderFooter(envelope.ByHeaderFooter); err != nil {
+return err
+}
+if err := f.validateColumns(envelope.Columns); err != nil {
+return err
+}
+}
+if !targetSeen {
+return f.FmtErr("missing target envelope")
+}
+return nil
+}
+
+func (f *fixedLengthFileFormat) validateByHeaderFooter(decl *byHeaderFooterDecl) error {
+if decl == nil {
+return nil
+}
+_, err := caches.GetRegex(decl.Header)
+if err != nil {
+return f.FmtErr("invalid 'header' regex '%s': %s", decl.Header, err.Error())
+}
+_, err = caches.GetRegex(decl.Footer)
+if err != nil {
+return f.FmtErr("invalid 'footer' regex '%s': %s", decl.Footer, err.Error())
+}
+return nil
+}
+
+func (f *fixedLengthFileFormat) validateColumns(cols []*columnDecl) error {
+columnNamesSeen := map[string]bool{}
+for _, col := range cols {
+if _, found := columnNamesSeen[col.Name]; found {
+return f.FmtErr("more than one column has the name '%s'", col.Name)
+}
+columnNamesSeen[col.Name] = true
+if col.LinePattern != nil {
+if _, err := caches.GetRegex(*col.LinePattern); err != nil {
+return f.FmtErr("invalid 'line_pattern' regex '%s': %s", *col.LinePattern, err.Error())
+}
+}
+}
+return nil
+}
+
+func (f *fixedLengthFileFormat) CreateFormatReader(
+name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
+// TODO
+_ = runtime.(*fixedLengthFormatRuntime)
+return nil, nil
+}
+
+func (f *fixedLengthFileFormat) FmtErr(format string, args ...interface{}) error {
+return fmt.Errorf("schema '%s': %s", f.schemaName, fmt.Sprintf(format, args...))
+}