Add a few more custom funcs plus a 'javascript'-vs-'eval'-vs-'ifese' benchmark comparison; add sub-sec support in times. (#58)

* Add a few more custom funcs plus a 'javascript'-vs-'eval'-vs-'ifese' benchmark comparison * add sub-second support into timetrie and parse

Author: jf-tech

cache/regexCache.go

@@ -0,0 +1,24 @@
+package cache
+
+import (
+"regexp"
+)
+
+// RegexCache is the default loading cache used for caching the compiled
+// regex expression. If the default size is too big/small and/or a cache limit isn't
+// desired at all, caller can simply replace the cache during global initialization.
+// But be aware it's global so any packages uses this package inside your process will
+// be affected.
+var RegexCache = NewLoadingCache()
+
+// GetRegex compiles a given regex pattern and returns a compiled *regexp.Regexp
+// or error.
+func GetRegex(pattern string) (*regexp.Regexp, error) {
+exp, err := RegexCache.Get(pattern, func(key interface{}) (interface{}, error) {
+return regexp.Compile(key.(string))
+})
+if err != nil {
+return nil, err
+}
+return exp.(*regexp.Regexp), nil
+}

cache/regexCache_test.go

@@ -0,0 +1,28 @@
+package cache
+
+import (
+"testing"
+
+"github.com/stretchr/testify/assert"
+)
+
+func TestGetRegex(t *testing.T) {
+RegexCache = NewLoadingCache()
+assert.Equal(t, 0, len(RegexCache.DumpForTest()))
+// failure case
+expr, err := GetRegex("[")
+assert.Error(t, err)
+assert.Equal(t, "error parsing regexp: missing closing ]: `[`", err.Error())
+assert.Nil(t, expr)
+assert.Equal(t, 0, len(RegexCache.DumpForTest()))
+// success case
+expr, err = GetRegex("^[0-9]{4}-[0-9]{2}-[0-9]{2}$")
+assert.NoError(t, err)
+assert.NotNil(t, expr)
+assert.Equal(t, 1, len(RegexCache.DumpForTest()))
+// repeat success case shouldn't case any cache growth
+expr, err = GetRegex("^[0-9]{4}-[0-9]{2}-[0-9]{2}$")
+assert.NoError(t, err)
+assert.NotNil(t, expr)
+assert.Equal(t, 1, len(RegexCache.DumpForTest()))
+}

cache/xpathExprCache.go

@@ -11,7 +11,7 @@ import (
 // be affected.
 var XPathExprCache = NewLoadingCache()
 
-// GetXPathExpr compiles a given xpath expression string and returns a compile xpath.Expr
+// GetXPathExpr compiles a given xpath expression string and returns a compiled *xpath.Expr
 // or error.
 func GetXPathExpr(expr string) (*xpath.Expr, error) {
 exp, err := XPathExprCache.Get(expr, func(key interface{}) (interface{}, error) {

omniparser/customfuncs/.snapshots/TestDumpBuiltinCustomFuncNames

@@ -1,6 +1,7 @@
 [
 "avg",
 "concat",
+"containsPattern",
 "dateTimeLayoutToRFC3339",
 "dateTimeToEpoch",
 "dateTimeToRFC3339",
@@ -9,10 +10,13 @@
 "eval",
 "external",
 "floor",
+"ifElse",
+"isEmpty",
 "javascript",
 "lower",
 "splitIntoJsonArray",
 "substring",
 "sum",
-"upper"
+"upper",
+"uuidv3"
 ]

omniparser/customfuncs/aggregate_test.go

@@ -8,116 +8,116 @@ import (
 
 func TestAvg(t *testing.T) {
 for _, test := range []struct {
-name  string
-inputs  []string
-expectedErr string
-expectedAvg string
+name string
+inputs []string
+err  string
+expected string
 }{
 {
-name:  "nil",
-inputs:  nil,
-expectedErr: "",
-expectedAvg: "0",
+name: "nil",
+inputs: nil,
+err:  "",
+expected: "0",
 },
 {
-name:  "empty",
-inputs:  []string{},
-expectedErr: "",
-expectedAvg: "0",
+name: "empty",
+inputs: []string{},
+err:  "",
+expected: "0",
 },
 {
-name:  "single",
-inputs:  []string{"3.14159265358"},
-expectedErr: "",
-expectedAvg: "3.14159265358",
+name: "single",
+inputs: []string{"3.14159265358"},
+err:  "",
+expected: "3.14159265358",
 },
 {
-name:  "multiple small ones",
-inputs:  []string{"3.45", "5.38"},
-expectedErr: "",
-expectedAvg: "4.415",
+name: "multiple small ones",
+inputs: []string{"3.45", "5.38"},
+err:  "",
+expected: "4.415",
 },
 {
-name:  "multiple big ones",
-inputs:  []string{"1.23e+9", "0.34E+10"},
-expectedErr: "",
-expectedAvg: "2.315e+09",
+name: "multiple big ones",
+inputs: []string{"1.23e+9", "0.34E+10"},
+err:  "",
+expected: "2.315e+09",
 },
 {
-name:  "invalid value",
-inputs:  []string{"1", "two"},
-expectedErr: `strconv.ParseFloat: parsing "two": invalid syntax`,
-expectedAvg: "",
+name: "invalid value",
+inputs: []string{"1", "two"},
+err:  `strconv.ParseFloat: parsing "two": invalid syntax`,
+expected: "",
 },
 } {
 t.Run(test.name, func(t *testing.T) {
 result, err := avg(nil, test.inputs...)
-if test.expectedErr != "" {
+if test.err != "" {
 assert.Error(t, err)
-assert.Equal(t, test.expectedErr, err.Error())
+assert.Equal(t, test.err, err.Error())
 assert.Equal(t, "", result)
 } else {
 
 assert.NoError(t, err)
-assert.Equal(t, test.expectedAvg, result)
+assert.Equal(t, test.expected, result)
 }
 })
 }
 }
 
 func TestSum(t *testing.T) {
 for _, test := range []struct {
-name  string
-inputs  []string
-expectedErr string
-expectedSum string
+name string
+inputs []string
+err  string
+expected string
 }{
 {
-name:  "nil",
-inputs:  nil,
-expectedErr: "",
-expectedSum: "0",
+name: "nil",
+inputs: nil,
+err:  "",
+expected: "0",
 },
 {
-name:  "empty",
-inputs:  []string{},
-expectedErr: "",
-expectedSum: "0",
+name: "empty",
+inputs: []string{},
+err:  "",
+expected: "0",
 },
 {
-name:  "single",
-inputs:  []string{"3.14159265358"},
-expectedErr: "",
-expectedSum: "3.14159265358",
+name: "single",
+inputs: []string{"3.14159265358"},
+err:  "",
+expected: "3.14159265358",
 },
 {
-name:  "multiple small ones",
-inputs:  []string{"3.45", "5.38"},
-expectedErr: "",
-expectedSum: "8.83",
+name: "multiple small ones",
+inputs: []string{"3.45", "5.38"},
+err:  "",
+expected: "8.83",
 },
 {
-name:  "multiple big ones",
-inputs:  []string{"1.23e+9", "0.34E+10"},
-expectedErr: "",
-expectedSum: "4.63e+09",
+name: "multiple big ones",
+inputs: []string{"1.23e+9", "0.34E+10"},
+err:  "",
+expected: "4.63e+09",
 },
 {
-name:  "invalid value",
-inputs:  []string{"1", "two"},
-expectedErr: `strconv.ParseFloat: parsing "two": invalid syntax`,
-expectedSum: "",
+name: "invalid value",
+inputs: []string{"1", "two"},
+err:  `strconv.ParseFloat: parsing "two": invalid syntax`,
+expected: "",
 },
 } {
 t.Run(test.name, func(t *testing.T) {
 result, err := sum(nil, test.inputs...)
-if test.expectedErr != "" {
+if test.err != "" {
 assert.Error(t, err)
-assert.Equal(t, test.expectedErr, err.Error())
+assert.Equal(t, test.err, err.Error())
 assert.Equal(t, "", result)
 } else {
 assert.NoError(t, err)
-assert.Equal(t, test.expectedSum, result)
+assert.Equal(t, test.expected, result)
 }
 })
 }

omniparser/customfuncs/customFuncs.go

@@ -8,6 +8,9 @@ import (
 "strconv"
 "strings"
 
+"github.com/google/uuid"
+
+"github.com/jf-tech/omniparser/cache"
 "github.com/jf-tech/omniparser/omniparser/transformctx"
 "github.com/jf-tech/omniparser/strs"
 )
@@ -34,24 +37,28 @@ var builtinPublishedCustomFuncs = map[string]CustomFuncType{
 // keep these custom funcs lexically sorted
 "avg": avg,
 "concat": concat,
+"containsPattern": containsPattern,
 "dateTimeLayoutToRFC3339": dateTimeLayoutToRFC3339,
 "dateTimeToEpoch": dateTimeToEpoch,
 "dateTimeToRFC3339": dateTimeToRFC3339,
+"eval": eval,
 "floor": floor,
+"ifElse": ifElse,
+"isEmpty": isEmpty,
 "javascript": javascript,
 "lower": lower,
+"splitIntoJsonArray": splitIntoJsonArray,
 "substring": substring,
 "sum": sum,
 "upper": upper,
+"uuidv3": uuidv3,
 }
 
 var builtinHiddenBackCompatCustomFuncs = map[string]CustomFuncType{
 // keep these custom funcs lexically sorted
 "dateTimeWithLayoutToRfc3339": dateTimeLayoutToRFC3339, // deprecated; use dateTimeLayoutToRFC3339.
 "dateTimeToRfc3339": dateTimeToRFC3339, // deprecated; use dateTimeToRFC3339.
-"eval": eval, // deprecated; use 'javascript'.
 "external": external, // deprecated; use "external" decl.
-"splitIntoJsonArray": splitIntoJsonArray, // deprecated; use 'javascript'.
 }
 
 // BuiltinCustomFuncs contains all the built-in custom functions.
@@ -65,6 +72,19 @@ func concat(_ *transformctx.Ctx, strs ...string) (string, error) {
 return b.String(), nil
 }
 
+func containsPattern(_ *transformctx.Ctx, regexPattern string, strs ...string) (string, error) {
+r, err := cache.GetRegex(regexPattern)
+if err != nil {
+return "", err
+}
+for _, str := range strs {
+if r.MatchString(str) {
+return "true", nil
+}
+}
+return "false", nil
+}
+
 func external(ctx *transformctx.Ctx, name string) (string, error) {
 if v, found := ctx.ExternalProperty(name); found {
 return v, nil
@@ -88,6 +108,30 @@ func floor(_ *transformctx.Ctx, value, decimalPlaces string) (string, error) {
 return fmt.Sprintf("%v", math.Floor(v*p10)/p10), nil
 }
 
+func ifElse(_ *transformctx.Ctx, conditionsAndValues ...string) (string, error) {
+if len(conditionsAndValues)%2 != 1 {
+return "", fmt.Errorf("arg number must be odd, but got: %d", len(conditionsAndValues))
+}
+for i := 0; i < len(conditionsAndValues)/2; i++ {
+condition, err := strconv.ParseBool(conditionsAndValues[2*i])
+if err != nil {
+return "", fmt.Errorf(
+`condition argument must be a boolean string, but got: %s`, conditionsAndValues[2*i])
+}
+if condition {
+return conditionsAndValues[(2*i)+1], nil
+}
+}
+return conditionsAndValues[len(conditionsAndValues)-1], nil
+}
+
+func isEmpty(_ *transformctx.Ctx, str string) (string, error) {
+if str == "" {
+return "true", nil
+}
+return "false", nil
+}
+
 func lower(_ *transformctx.Ctx, s string) (string, error) {
 return strings.ToLower(s), nil
 }
@@ -157,3 +201,7 @@ func substring(_ *transformctx.Ctx, str, startIndex, lengthStr string) (string,
 func upper(_ *transformctx.Ctx, s string) (string, error) {
 return strings.ToUpper(s), nil
 }
+
+func uuidv3(_ *transformctx.Ctx, s string) (string, error) {
+return uuid.NewMD5(uuid.Nil, []byte(s)).String(), nil
+}