Add csv fileformat to omniv2 handler + sample + a couple more custom_func (#88)

Add csv fileformat to omniv2 handler + sample + a couple more custom_func

Author: jf-tech

README.md

@@ -119,9 +119,12 @@ Take a detailed look at samples here:
  extension repo/package; the rest of the library is just golang 1.12.
 
 ## Recent Feature Additions
-- added trie based high performance `times.SmartParse`.
-- command line interface (one-off `transform` cmd or long-running http `server` mode).
-- javascript engine integration as a custom_func.
+- Added CSV file format support in omniv2 handler.
+- Introduced IDR node cache for allocation recycling. 
+- Introduced [IDR](./idr/README.md) for in-memory data representation.
+- Added trie based high performance `times.SmartParse`.
+- Command line interface (one-off `transform` cmd or long-running http `server` mode).
+- `javascript` engine integration as a custom_func.
 - JSON stream parser.
 - Extensibility:
  - Ability to provide custom functions.

cli/cmd/serverCmd.go

@@ -150,7 +150,7 @@ func httpPostTransform(w http.ResponseWriter, r *http.Request) {
 
 var (
 sampleDir = "../../samples/omniv2/"
-sampleFormats = []string{"json", "xml"}
+sampleFormats = []string{"csv", "json", "xml"}
 sampleInputFilenamePattern = regexp.MustCompile("^([0-9]+[_a-zA-Z]+)\\.input\\.[a-z]+$")
 )
 

customfuncs/.snapshots/TestDumpBuiltinCustomFuncNames

@@ -19,6 +19,8 @@
 "splitIntoJsonArray",
 "substring",
 "sum",
+"switch",
+"switchByPattern",
 "upper",
 "uuidv3"
 ]

customfuncs/customFuncs.go

@@ -4,6 +4,7 @@ import (
 "encoding/json"
 "fmt"
 "math"
+"regexp"
 "strconv"
 "strings"
 
@@ -50,6 +51,8 @@ var builtinPublishedCustomFuncs = map[string]CustomFuncType{
 "splitIntoJsonArray": splitIntoJsonArray,
 "substring": substring,
 "sum": sum,
+"switch": switchFunc,
+"switchByPattern": switchByPattern,
 "upper": upper,
 "uuidv3": uuidv3,
 }
@@ -208,6 +211,36 @@ func substring(_ *transformctx.Ctx, str, startIndex, lengthStr string) (string,
 return string(runes[start : start+length]), nil
 }
 
+func switchFunc(ctx *transformctx.Ctx, expr string, casesReturns ...string) (string, error) {
+if len(casesReturns)%2 != 1 {
+return "", fmt.Errorf("length of 'casesReturns' must be odd, but got: %d", len(casesReturns))
+}
+patternsReturns := make([]string, len(casesReturns))
+for i := 0; i < len(patternsReturns)/2; i++ {
+patternsReturns[2*i] = "^" + regexp.QuoteMeta(casesReturns[2*i]) + "$"
+patternsReturns[2*i+1] = casesReturns[2*i+1]
+}
+patternsReturns[len(casesReturns)-1] = casesReturns[len(casesReturns)-1]
+return switchByPattern(ctx, expr, patternsReturns...)
+}
+
+func switchByPattern(_ *transformctx.Ctx, expr string, patternsReturns ...string) (string, error) {
+if len(patternsReturns)%2 != 1 {
+return "", fmt.Errorf(
+"length of 'patternsReturns' must be odd, but got: %d", len(patternsReturns))
+}
+for i := 0; i < len(patternsReturns)/2; i++ {
+re, err := caches.GetRegex(patternsReturns[2*i])
+if err != nil {
+return "", fmt.Errorf(`invalid pattern '%s', err: %s`, patternsReturns[2*i], err.Error())
+}
+if re.MatchString(expr) {
+return patternsReturns[(2*i)+1], nil
+}
+}
+return patternsReturns[len(patternsReturns)-1], nil
+}
+
 func upper(_ *transformctx.Ctx, s string) (string, error) {
 return strings.ToUpper(s), nil
 }

customfuncs/customFuncs_test.go

@@ -655,6 +655,112 @@ func TestSubstring(t *testing.T) {
 }
 }
 
+func TestSwitchFunc(t *testing.T) {
+for _, test := range []struct {
+name string
+expr string
+casesReturns []string
+err string
+expected string
+}{
+{
+name: "empty casesReturns",
+expr: "abc",
+casesReturns: nil,
+err: "length of 'casesReturns' must be odd, but got: 0",
+},
+{
+name: "even casesReturns length",
+expr: "abc",
+casesReturns: []string{"1", "2", "3", "4"},
+err: "length of 'casesReturns' must be odd, but got: 4",
+},
+{
+name: "no case, just default",
+expr: "abc",
+casesReturns: []string{"default"},
+expected: "default",
+},
+{
+name: "case string contains special characters",
+expr: "How do you do",
+casesReturns: []string{
+"How do you do?", "Wrong",
+"How do you do", "Correct",
+"Huh"},
+expected: "Correct",
+},
+} {
+t.Run(test.name, func(t *testing.T) {
+result, err := switchFunc(nil, test.expr, test.casesReturns...)
+if test.err != "" {
+assert.Error(t, err)
+assert.Equal(t, test.err, err.Error())
+assert.Equal(t, "", result)
+} else {
+assert.NoError(t, err)
+assert.Equal(t, test.expected, result)
+}
+})
+}
+}
+
+func TestSwitchByPattern(t *testing.T) {
+for _, test := range []struct {
+name string
+expr string
+patternsReturns []string
+err string
+expected string
+}{
+{
+name: "empty patternsReturns",
+expr: "abc",
+patternsReturns: nil,
+err: "length of 'patternsReturns' must be odd, but got: 0",
+},
+{
+name: "even patternsReturns length",
+expr: "abc",
+patternsReturns: []string{"1", "2", "3", "4"},
+err: "length of 'patternsReturns' must be odd, but got: 4",
+},
+{
+name: "regex invalid",
+expr: "abc",
+patternsReturns: []string{"[", "2", "3"},
+err: "invalid pattern '[', err: error parsing regexp: missing closing ]: `[`",
+},
+{
+name: "no pattern, only default",
+expr: "abc",
+patternsReturns: []string{"default"},
+expected: "default",
+},
+{
+name: "case string contains special characters",
+expr: "2019/02/23",
+patternsReturns: []string{
+"^[0-9]{2}/[0-9]{2}/[0-9]{4}$", "Wrong",
+"^[0-9]{4}/[0-9]{2}/[0-9]{2}$", "Correct",
+"Huh"},
+expected: "Correct",
+},
+} {
+t.Run(test.name, func(t *testing.T) {
+result, err := switchByPattern(nil, test.expr, test.patternsReturns...)
+if test.err != "" {
+assert.Error(t, err)
+assert.Equal(t, test.err, err.Error())
+assert.Equal(t, "", result)
+} else {
+assert.NoError(t, err)
+assert.Equal(t, test.expected, result)
+}
+})
+}
+}
+
 func TestUpper(t *testing.T) {
 s, err := upper(nil, "")
 assert.NoError(t, err)

handlers/omni/v2/fileformat/csv/.snapshots/TestValidateSchema-success_1

@@ -0,0 +1,19 @@
+{
+"file_declaration": {
+"delimiter": ",",
+"replace_double_quotes": true,
+"header_row_index": 2,
+"data_row_index": 4,
+"columns": [
+{
+"name": "col1",
+"alias": null
+},
+{
+"name": "col 2",
+"alias": "col2"
+}
+]
+},
+"XPath": ".[col1 != 'skip']"
+}

handlers/omni/v2/fileformat/csv/.snapshots/TestValidateSchema-success_2

@@ -0,0 +1,15 @@
+{
+"file_declaration": {
+"delimiter": ",",
+"replace_double_quotes": false,
+"header_row_index": null,
+"data_row_index": 1,
+"columns": [
+{
+"name": "col1",
+"alias": null
+}
+]
+},
+"XPath": ""
+}

handlers/omni/v2/fileformat/csv/format.go

@@ -0,0 +1,107 @@
+package omniv2csv
+
+import (
+"encoding/json"
+"fmt"
+"io"
+"strings"
+
+"github.com/jf-tech/go-corelib/caches"
+"github.com/jf-tech/go-corelib/strs"
+
+"github.com/jf-tech/omniparser/errs"
+omniv2fileformat "github.com/jf-tech/omniparser/handlers/omni/v2/fileformat"
+"github.com/jf-tech/omniparser/handlers/omni/v2/transform"
+"github.com/jf-tech/omniparser/validation"
+)
+
+const (
+fileFormatCSV = "csv"
+fileFormatDelimited = "delimited"
+)
+
+type csvFileFormat struct {
+schemaName string
+}
+
+// NewCSVFileFormat creates a FileFormat for CSV for omniv2 schema handler.
+func NewCSVFileFormat(schemaName string) omniv2fileformat.FileFormat {
+return &csvFileFormat{schemaName: schemaName}
+}
+
+type csvFormatRuntime struct {
+Decl *fileDecl `json:"file_declaration"`
+XPath string
+}
+
+func (f *csvFileFormat) ValidateSchema(
+format string, schemaContent []byte, finalOutputDecl *transform.Decl) (interface{}, error) {
+if format != fileFormatCSV && format != fileFormatDelimited {
+return nil, errs.ErrSchemaNotSupported
+}
+err := validation.SchemaValidate(f.schemaName, schemaContent, validation.JSONSchemaCSVFileDeclaration)
+if err != nil {
+// err is already context formatted.
+return nil, err
+}
+var runtime csvFormatRuntime
+_ = json.Unmarshal(schemaContent, &runtime) // JSON schema validation earlier guarantees Unmarshal success.
+err = f.validateFileDecl(runtime.Decl)
+if err != nil {
+// err is already context formatted.
+return nil, err
+}
+if finalOutputDecl == nil {
+return nil, f.FmtErr("'FINAL_OUTPUT' is missing")
+}
+runtime.XPath = strings.TrimSpace(strs.StrPtrOrElse(finalOutputDecl.XPath, ""))
+if runtime.XPath != "" {
+_, err := caches.GetXPathExpr(runtime.XPath)
+if err != nil {
+return nil, f.FmtErr("'FINAL_OUTPUT.xpath' (value: '%s') is invalid, err: %s",
+runtime.XPath, err.Error())
+}
+}
+return &runtime, nil
+}
+
+func (f *csvFileFormat) validateFileDecl(decl *fileDecl) error {
+// If header_row_index is specified, then it must be < data_row_index
+if decl.HeaderRowIndex != nil && *decl.HeaderRowIndex >= decl.DataRowIndex {
+return f.FmtErr(
+"file_declaration.header_row_index(%d) must be smaller than file_declaration.data_row_index(%d)",
+*decl.HeaderRowIndex, decl.DataRowIndex)
+}
+if err := f.validateColumns(decl.Columns); err != nil {
+return err
+}
+return nil
+}
+
+func (f *csvFileFormat) validateColumns(columns []column) error {
+namesSeen := map[string]bool{}
+aliasesSeen := map[string]bool{}
+for _, column := range columns {
+if _, found := namesSeen[column.Name]; found {
+return f.FmtErr("file_declaration.columns contains duplicate name '%s'", column.Name)
+}
+namesSeen[column.Name] = true
+if column.Alias != nil {
+if _, found := aliasesSeen[*column.Alias]; found {
+return f.FmtErr("file_declaration.columns contains duplicate alias '%s'", *column.Alias)
+}
+aliasesSeen[*column.Alias] = true
+}
+}
+return nil
+}
+
+func (f *csvFileFormat) CreateFormatReader(
+name string, r io.Reader, runtime interface{}) (omniv2fileformat.FormatReader, error) {
+csv := runtime.(*csvFormatRuntime)
+return NewReader(name, r, csv.Decl, csv.XPath)
+}
+
+func (f *csvFileFormat) FmtErr(format string, args ...interface{}) error {
+return fmt.Errorf("schema '%s': %s", f.schemaName, fmt.Sprintf(format, args...))
+}