A lot of memory allocation optimizations (go-mysql-org#466)

A lot of memory allocation optimizations: zero-allocation in hot paths for mysql/client. Changed Resultset.Values public API.

Author: atercattus

client/req.go

@@ -1,5 +1,9 @@
 package client
 
+import (
+"github.com/siddontang/go-mysql/utils"
+)
+
 func (c *Conn) writeCommand(command byte) error {
 c.ResetSequence()
 
@@ -16,28 +20,20 @@ func (c *Conn) writeCommandBuf(command byte, arg []byte) error {
 c.ResetSequence()
 
 length := len(arg) + 1
-
-data := make([]byte, length+4)
-
+data := utils.ByteSliceGet(length + 4)
 data[4] = command
 
 copy(data[5:], arg)
 
-return c.WritePacket(data)
-}
-
-func (c *Conn) writeCommandStr(command byte, arg string) error {
-c.ResetSequence()
-
-length := len(arg) + 1
-
-data := make([]byte, length+4)
+err := c.WritePacket(data)
 
-data[4] = command
+utils.ByteSlicePut(data)
 
-copy(data[5:], arg)
+return err
+}
 
-return c.WritePacket(data)
+func (c *Conn) writeCommandStr(command byte, arg string) error {
+return c.writeCommandBuf(command, utils.StringToByteSlice(arg))
 }
 
 func (c *Conn) writeCommandUint32(command byte, arg uint32) error {

client/resp.go

@@ -1,15 +1,15 @@
 package client
 
 import (
-"encoding/binary"
-
 "bytes"
 "crypto/rsa"
 "crypto/x509"
+"encoding/binary"
 "encoding/pem"
 
 "github.com/pingcap/errors"
 . "github.com/siddontang/go-mysql/mysql"
+"github.com/siddontang/go-mysql/utils"
 "github.com/siddontang/go/hack"
 )
 
@@ -212,40 +212,35 @@ func (c *Conn) readOK() (*Result, error) {
 }
 
 func (c *Conn) readResult(binary bool) (*Result, error) {
-data, err := c.ReadPacket()
+firstPkgBuf, err := c.ReadPacketReuseMem(utils.ByteSliceGet(16)[:0])
+defer utils.ByteSlicePut(firstPkgBuf)
+
 if err != nil {
 return nil, errors.Trace(err)
 }
 
-if data[0] == OK_HEADER {
-return c.handleOKPacket(data)
-} else if data[0] == ERR_HEADER {
-return nil, c.handleErrorPacket(data)
-} else if data[0] == LocalInFile_HEADER {
+if firstPkgBuf[0] == OK_HEADER {
+return c.handleOKPacket(firstPkgBuf)
+} else if firstPkgBuf[0] == ERR_HEADER {
+return nil, c.handleErrorPacket(append([]byte{}, firstPkgBuf...))
+} else if firstPkgBuf[0] == LocalInFile_HEADER {
 return nil, ErrMalformPacket
 }
 
-return c.readResultset(data, binary)
+return c.readResultset(firstPkgBuf, binary)
 }
 
 func (c *Conn) readResultset(data []byte, binary bool) (*Result, error) {
-result := &Result{
-Status: 0,
-InsertId: 0,
-AffectedRows: 0,
-
-Resultset: &Resultset{},
-}
-
 // column count
 count, _, n := LengthEncodedInt(data)
 
 if n-len(data) != 0 {
 return nil, ErrMalformPacket
 }
 
-result.Fields = make([]*Field, count)
-result.FieldNames = make(map[string]int, count)
+result := &Result{
+Resultset: NewResultset(int(count)),
+}
 
 if err := c.readResultColumns(result); err != nil {
 return nil, errors.Trace(err)
@@ -263,10 +258,12 @@ func (c *Conn) readResultColumns(result *Result) (err error) {
 var data []byte
 
 for {
-data, err = c.ReadPacket()
+rawPkgLen := len(result.RawPkg)
+result.RawPkg, err = c.ReadPacketReuseMem(result.RawPkg)
 if err != nil {
 return
 }
+data = result.RawPkg[rawPkgLen:]
 
 // EOF Packet
 if c.isEOFPacket(data) {
@@ -284,7 +281,10 @@ func (c *Conn) readResultColumns(result *Result) (err error) {
 return
 }
 
-result.Fields[i], err = FieldData(data).Parse()
+if result.Fields[i] == nil {
+result.Fields[i] = &Field{}
+}
+err = result.Fields[i].Parse(data)
 if err != nil {
 return
 }
@@ -299,11 +299,12 @@ func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) {
 var data []byte
 
 for {
-data, err = c.ReadPacket()
-
+rawPkgLen := len(result.RawPkg)
+result.RawPkg, err = c.ReadPacketReuseMem(result.RawPkg)
 if err != nil {
 return
 }
+data = result.RawPkg[rawPkgLen:]
 
 // EOF Packet
 if c.isEOFPacket(data) {
@@ -324,10 +325,14 @@ func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) {
 result.RowDatas = append(result.RowDatas, data)
 }
 
-result.Values = make([][]interface{}, len(result.RowDatas))
+if cap(result.Values) < len(result.RowDatas) {
+result.Values = make([][]FieldValue, len(result.RowDatas))
+} else {
+result.Values = result.Values[:len(result.RowDatas)]
+}
 
 for i := range result.Values {
-result.Values[i], err = result.RowDatas[i].Parse(result.Fields, isBinary)
+result.Values[i], err = result.RowDatas[i].Parse(result.Fields, isBinary, result.Values[i])
 
 if err != nil {
 return errors.Trace(err)

driver/dirver_test.go renamed to driver/driver_test.go

@@ -2,6 +2,8 @@ package mysql
 
 import (
 "encoding/binary"
+
+"github.com/siddontang/go-mysql/utils"
 )
 
 type FieldData []byte
@@ -23,9 +25,23 @@ type Field struct {
 DefaultValue []byte
 }
 
-func (p FieldData) Parse() (f *Field, err error) {
-f = new(Field)
+type FieldValueType uint8
+
+type FieldValue struct {
+Type FieldValueType
+value uint64 // Also for int64 and float64
+str []byte
+}
+
+const (
+FieldValueTypeNull = iota
+FieldValueTypeUnsigned
+FieldValueTypeSigned
+FieldValueTypeFloat
+FieldValueTypeString
+)
 
+func (f *Field) Parse(p FieldData) (err error) {
 f.Data = p
 
 var n int
@@ -117,6 +133,14 @@ func (p FieldData) Parse() (f *Field, err error) {
 return
 }
 
+func (p FieldData) Parse() (f *Field, err error) {
+f = new(Field)
+if err = f.Parse(p); err != nil {
+return nil, err
+}
+return f, nil
+}
+
 func (f *Field) Dump() []byte {
 if f == nil {
 f = &Field{}
@@ -155,3 +179,34 @@ func (f *Field) Dump() []byte {
 
 return data
 }
+
+func (fv *FieldValue) AsUint64() uint64 {
+return fv.value
+}
+
+func (fv *FieldValue) AsInt64() int64 {
+return utils.Uint64ToInt64(fv.value)
+}
+
+func (fv *FieldValue) AsFloat64() float64 {
+return utils.Uint64ToFloat64(fv.value)
+}
+
+func (fv *FieldValue) AsString() []byte {
+return fv.str
+}
+
+func (fv *FieldValue) Value() interface{} {
+switch fv.Type {
+case FieldValueTypeUnsigned:
+return fv.AsUint64()
+case FieldValueTypeSigned:
+return fv.AsInt64()
+case FieldValueTypeFloat:
+return fv.AsFloat64()
+case FieldValueTypeString:
+return fv.AsString()
+default: // FieldValueTypeNull
+return nil
+}
+}

mysql/field.go

@@ -12,3 +12,10 @@ type Result struct {
 type Executer interface {
 Execute(query string, args ...interface{}) (*Result, error)
 }
+
+func (r *Result) Close() {
+if r.Resultset != nil {
+r.Resultset.returnToPool()
+r.Resultset = nil
+}
+}