Init commit

Author: willzhen

.gitignore

@@ -0,0 +1 @@
+go.sum

README.md

@@ -0,0 +1,35 @@
+# GO library
+- client：调用第三方服务的客户端。
+ - ti: ti 平台客户端。
+ - ti3.4
+ - ti3.7
+- httpcall：http 调用的封装。
+ - json_post.go: JsonPost。
+ - json_get.go: JsonGet。
+ - json_put.go: JsonPut。
+ - json_delete.go: JsonDelete。
+- misc：常用的公共函数库
+ - time_related：时间相关的函数
+ - time_tick.go: TimeTick -- 计时器。
+ - timed.go: Timed -- 函数计时调用。
+ - goroutine_help.go：
+ - GoroutineHelp.Recoverd：协程 recover。
+ - GoroutineHelp.SafeGo：安全的 go。
+ - retry.go： Retry -- 带重试调用。
+ - id.go：
+ - GenerateRandomString：生成随机字符串。
+ - GenerateUUID：生成UUID。
+ - BighumpToUnderscore：大驼峰参数转换成下划线。
+ - UnderscoreToBighump：下划线参数转换成大驼峰。
+ - log.og
+ - FoldLog: 折叠日志中数据过长的字段，仅支持 json。
+- stl_extension：stl 的扩展
+ - limit_waitgroup.go：LimitWaitGroup -- 对于系统 WaitGroup 的扩展，支持 limit 并发限制并且阻塞。
+ - ordered_map.go：OrderedMap -- 实现了 c++ 中的排序 map，可以按照顺序遍历所有元素。
+- text: 文本处理相关方法
+ - aho_automaton.go：ac 自动机，多模式串快速匹配。在一个文本中找出来，出现过哪些字符串子串以及其定位。可以理解对同一文本 s 多次调用 strings.Contains(s, xxx) 的加速。
+ - BuildAcTrie: 构建自动机。
+ - AcTrie.Search: 匹配。
+ - edit_distance.go：
+ - Levenshtein：文本编辑距离。
+ - TextSim：计算两个文本的相似度。

go.mod

@@ -0,0 +1,10 @@
+module github.com/memory-overflow/go-common-library
+
+go 1.16
+
+require (
+github.com/eapache/queue v1.1.0
+github.com/google/uuid v1.3.0
+github.com/json-iterator/go v1.1.12
+github.com/tidwall/gjson v1.14.4
+)

httpcall/httpcall_test.go

@@ -0,0 +1,27 @@
+package httpcall_test
+
+import (
+"context"
+"testing"
+"time"
+
+"github.com/memory-overflow/go-common-library/httpcall"
+service "github.com/memory-overflow/go-common-library/task_scheduler/example/add_service"
+)
+
+func TestJsonPost(t *testing.T) {
+// 控制请求超时时间
+ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+defer cancel()
+uri := "http://127.0.0.1:8000/add"
+req := &service.AddRequest{
+A: 1,
+B: 2,
+}
+rsp := &service.AddResponse{}
+err := httpcall.JsonPost(ctx, uri, nil, req, rsp)
+if err != nil {
+t.Fatalf("JsonPost err: %v", err)
+}
+t.Logf("add taskId: %d", rsp.TaskId)
+}

httpcall/json_delete.go

@@ -0,0 +1,42 @@
+package httpcall
+
+import (
+"bytes"
+"context"
+"encoding/json"
+"fmt"
+"io/ioutil"
+"net/http"
+)
+
+// JsonDelete http delete 请求
+func JsonDelete(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+body, _ := json.Marshal(req)
+reqCtx, err := http.NewRequestWithContext(ctx, http.MethodDelete, uri, bytes.NewReader(body))
+if err != nil {
+return fmt.Errorf("JsonDelete make NewRequest error: %v", err)
+}
+
+for head, value := range header {
+reqCtx.Header.Set(head, value)
+}
+
+response, err := http.DefaultClient.Do(reqCtx)
+if err != nil {
+return fmt.Errorf("JsonDelete do request error: %v", err)
+}
+defer response.Body.Close()
+if response.StatusCode != http.StatusOK {
+return fmt.Errorf("JsonDelete response status error, statuscode: %d, reason: %s",
+response.StatusCode, response.Status)
+}
+data, _ := ioutil.ReadAll(response.Body)
+// log.Printf("JsonDelete req: %v, rsp: %v \n", string(body), string(data))
+if rsp != nil {
+err = json.Unmarshal(data, rsp)
+if err != nil {
+return fmt.Errorf("JsonDelete response protocol error: %v, data: %s", err, string(data))
+}
+}
+return nil
+}

httpcall/json_get.go

@@ -0,0 +1,39 @@
+package httpcall
+
+import (
+"context"
+"encoding/json"
+"fmt"
+"io/ioutil"
+"net/http"
+)
+
+// JsonGet http json get 请求
+func JsonGet(ctx context.Context, uri string, header map[string]string, rsp interface{}) (err error) {
+reqCtx, err := http.NewRequestWithContext(ctx, http.MethodGet, uri, nil)
+if err != nil {
+return fmt.Errorf("JsonGet make NewRequest error: %v", err)
+}
+
+for head, value := range header {
+reqCtx.Header.Set(head, value)
+}
+
+response, err := http.DefaultClient.Do(reqCtx)
+if err != nil {
+return fmt.Errorf("JsonGet do request error: %v", err)
+}
+defer response.Body.Close()
+if response.StatusCode != http.StatusOK {
+return fmt.Errorf("JsonGet response status error, statuscode: %d, reason: %s",
+response.StatusCode, response.Status)
+}
+data, _ := ioutil.ReadAll(response.Body)
+if rsp != nil {
+err = json.Unmarshal(data, rsp)
+if err != nil {
+return fmt.Errorf("JsonGet response protocol error: %v, data: %s", err, string(data))
+}
+}
+return nil
+}

httpcall/json_post.go

@@ -0,0 +1,44 @@
+package httpcall
+
+import (
+"bytes"
+"context"
+"encoding/json"
+"fmt"
+"io/ioutil"
+"net/http"
+)
+
+// JsonPost http json post 请求
+func JsonPost(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+body, _ := json.Marshal(req)
+reqCtx, err := http.NewRequestWithContext(ctx, http.MethodPost, uri, bytes.NewReader(body))
+if err != nil {
+return fmt.Errorf("HttpPost make NewRequest error: %v", err)
+}
+if header == nil {
+reqCtx.Header.Set("Content-Type", "application/json")
+} else {
+for head, value := range header {
+reqCtx.Header.Set(head, value)
+}
+}
+response, err := http.DefaultClient.Do(reqCtx)
+if err != nil {
+return fmt.Errorf("HttpPost do request error: %v", err)
+}
+defer response.Body.Close()
+if response.StatusCode != http.StatusOK {
+return fmt.Errorf("HttpPost response status error, statuscode: %d, reason: %s",
+response.StatusCode, response.Status)
+}
+data, _ := ioutil.ReadAll(response.Body)
+// log.Printf("JsonPost req: %v, rsp: %v \n", string(body), string(data))
+if rsp != nil {
+err = json.Unmarshal(data, rsp)
+if err != nil {
+return fmt.Errorf("HttpPost response protocol error: %v, data: %s", err, string(data))
+}
+}
+return nil
+}

httpcall/json_put.go

@@ -0,0 +1,47 @@
+package httpcall
+
+import (
+"bytes"
+"context"
+"encoding/json"
+"fmt"
+"io/ioutil"
+"net/http"
+)
+
+// JsonPut http put 请求
+func JsonPut(ctx context.Context, uri string, header map[string]string, req, rsp interface{}) (err error) {
+body, ok := req.([]byte)
+if !ok {
+body, _ = json.Marshal(req)
+}
+reqCtx, err := http.NewRequestWithContext(ctx, http.MethodPut, uri, bytes.NewReader(body))
+if err != nil {
+return fmt.Errorf("JsonPut make NewRequest error: %v", err)
+}
+if header == nil {
+reqCtx.Header.Set("Content-Type", "application/octet-stream")
+} else {
+for head, value := range header {
+reqCtx.Header.Set(head, value)
+}
+}
+response, err := http.DefaultClient.Do(reqCtx)
+if err != nil {
+return fmt.Errorf("JsonPut do request error: %v", err)
+}
+defer response.Body.Close()
+if response.StatusCode != http.StatusOK {
+return fmt.Errorf("JsonPut response status error, statuscode: %d, reason: %s",
+response.StatusCode, response.Status)
+}
+data, _ := ioutil.ReadAll(response.Body)
+// log.Printf("JsonPut req: %v, rsp: %v \n", string(body), string(data))
+if rsp != nil {
+err = json.Unmarshal(data, rsp)
+if err != nil {
+return fmt.Errorf("JsonPut response protocol error: %v, data: %s", err, string(data))
+}
+}
+return nil
+}

httpcall/readme.md

@@ -0,0 +1,35 @@
+# http 最常用的 json 协议调用封装
+- json_post.go: JsonPost。
+- json_get.go: JsonGet。
+- json_put.go: JsonPut。
+- json_delete.go: JsonDelete。
+
+
+example: https://github.com/memory-overflow/go-common-library/httpcall/httpcall_test.go
+```go
+import (
+"context"
+"testing"
+"time"
+
+"github.com/memory-overflow/go-common-library/httpcall"
+service "github.com/memory-overflow/go-common-library/task_scheduler/example/add_service"
+)
+
+func TestJsonPost(t *testing.T) {
+// 控制请求超时时间
+ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+defer cancel()
+uri := "http://127.0.0.1:8000/add"
+req := &service.AddRequest{
+A: 1,
+B: 2,
+}
+rsp := &service.AddResponse{}
+err := httpcall.JsonPost(ctx, uri, nil, req, rsp)
+if err != nil {
+t.Fatalf("JsonPost err: %v", err)
+}
+t.Logf("add taskId: %d", rsp.TaskId)
+}
+```

misc/donwlod.go

@@ -0,0 +1,88 @@
+package misc
+
+import (
+"context"
+"errors"
+"fmt"
+"io"
+"net/http"
+"os"
+"strconv"
+)
+
+// DownloadFile 下载网络文件到服务器本地
+func DownloadFile(ctx context.Context, url string, dirs, fileName string) error {
+req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+if err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+res, err := http.DefaultClient.Do(req)
+if err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+if res.StatusCode != 200 {
+return errors.New("DownloadFile " + url + " failed! " + res.Status)
+}
+if err := os.MkdirAll(dirs, 0766); err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+filePath := dirs + "/" + fileName
+file, err := os.Create(filePath)
+if err != nil {
+return errors.New("DownloadFile " + url + " create file failed! " + err.Error())
+}
+if length, err := io.Copy(file, res.Body); err != nil {
+os.Remove(filePath)
+return errors.New("DownloadFile " + url + " copy failed! " + err.Error() +
+" copied length: " + strconv.Itoa(int(length)))
+}
+file.Close()
+_, err = os.Stat(filePath)
+if err != nil {
+os.Remove(filePath)
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+return nil
+}
+
+// DownloadFileWithLimit 下载网络文件到服务器本地，限制文件大小
+func DownloadFileWithLimit(ctx context.Context, url string, dirs, fileName string, maxBytes int64) error {
+req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+if err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+res, err := http.DefaultClient.Do(req)
+if err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+if res.StatusCode != 200 {
+return errors.New("DownloadFile " + url + " failed! " + res.Status)
+}
+if err := os.MkdirAll(dirs, 0766); err != nil {
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+filePath := dirs + "/" + fileName
+file, err := os.Create(filePath)
+if err != nil {
+return errors.New("DownloadFile " + url + " create file failed! " + err.Error())
+}
+if length, err := io.CopyN(file, res.Body, int64(maxBytes)); err != nil && err != io.EOF {
+os.Remove(filePath)
+return errors.New("DownloadFile " + url + " copy failed! " + err.Error() +
+" copied length: " + strconv.Itoa(int(length)))
+}
+file.Close()
+nextByte := make([]byte, 1)
+nRead, _ := io.ReadFull(res.Body, nextByte)
+if nRead > 0 {
+// Yep, there's too much data
+os.Remove(filePath)
+return fmt.Errorf("DownloadFile %s exceed the file size limit %d", url, maxBytes)
+}
+_, err = os.Stat(filePath)
+if err != nil {
+os.Remove(filePath)
+return errors.New("DownloadFile " + url + " failed! " + err.Error())
+}
+return nil
+}