fixes

Author: sigalikanevsky

canal/canal_test.go

@@ -388,13 +388,10 @@ func TestGenerateCharsetQuery(t *testing.T) {
 expected := `
 SELECT 
  c.ORDINAL_POSITION,
-COALESCE(
-c.CHARACTER_SET_NAME,
-CASE 
-WHEN c.DATA_TYPE IN ('binary','varbinary','tinyblob','blob','mediumblob','longblob') THEN col.CHARACTER_SET_NAME
-ELSE NULL
-END
-) AS CHARACTER_SET_NAME,
+CASE 
+WHEN c.CHARACTER_SET_NAME IS NOT NULL THEN c.CHARACTER_SET_NAME
+WHEN c.DATA_TYPE IN ('binary','varbinary','tinyblob','blob','mediumblob','longblob') THEN col.CHARACTER_SET_NAME
+END AS CHARACTER_SET_NAME,
 c.COLUMN_NAME
 FROM 
 information_schema.COLUMNS c

cmd/go-binlogparser/.!93441!main.go

@@ -22,6 +22,7 @@ import (
 "strconv"
 "strings"
 "time"
+"unicode/utf8"
 )
 
 var errMissingTableMapEvent = errors.New("invalid table id, no corresponding table map event")
@@ -1195,8 +1196,12 @@ func convertToString(s interface{}) (string, bool) {
 }
 switch v := s.(type) {
 case []uint8:
-str := sanitizeNonPrintable(string(v))
-return str, true
+// Only convert to string if the bytes are valid UTF-8; otherwise keep as bytes
+if utf8.Valid(v) {
+str := string(v)
+return str, true
+}
+return "", false
 default:
 return "", false
 }
@@ -1206,16 +1211,13 @@ func decodeStringByCharSet(data []byte, charset string, length int) (v string, n
 enc, err := getDecoderByCharsetName(charset)
 if err != nil {
 log.Errorf(err.Error())
-v, n = decodeString(data, length)
-return sanitizeNonPrintable(v), n
+return decodeString(data, length)
 }
 if enc == nil {
 log.Warnf("Falling back to default decoding for charset: %s", charset)
-v, n = decodeString(data, length)
-return sanitizeNonPrintable(v), n
+return decodeString(data, length)
 }
-v, n = decodeStringWithEncoder(data, length, enc)
-return sanitizeNonPrintable(v), n
+return decodeStringWithEncoder(data, length, enc)
 }
 
 // sanitizeNonPrintable replaces non-printable runes with space for readability.

replication/row_event.go

@@ -1614,27 +1614,30 @@ func TestDecodeByCharSet(t *testing.T) {
 }
 }
 
-func TestDecodeValueBinaryLatin1Fallback(t *testing.T) {
+func TestDecodeValueBinaryFallback(t *testing.T) {
 // Simulate VARCHAR/VAR_STRING with charset binary and invalid UTF-8
 e := &RowsEvent{}
 // Length-encoded: 1-byte length + bytes
 // bytes correspond to expected visual string "1rHuÎ ð XÅXÄ " after mapping
 // include two leading control bytes [0x86, 0x11] which should become spaces,
 // and a trailing 0x01 which also becomes a space
 raw := []byte{0x11, 0x86, 0x11, '1', 'r', 'H', 'u', 0xce, 0x20, 0xf0, 0x20, 0x20, 0x20, 'X', 0xc5, 'X', 0xc4, 0x01}
-v, n, err := e.decodeValue(raw, mysql.MYSQL_TYPE_VAR_STRING, "latin1", 255)
-if err != nil {
-t.Fatalf("decodeValue error: %v", err)
-}
-if n != int(raw[0])+1 {
-t.Fatalf("read length = %d, want %d", n, int(raw[0])+1)
-}
-s, ok := v.(string)
-if !ok {
-t.Fatalf("value type = %T, want string", v)
-}
-want := " 1rHuÎ ð XÅXÄ "
-if s != want {
-t.Fatalf("decoded string = %q, want %q", s, want)
+charsets := []string{"latin1", "utf8", "utf8mb4", "cp1251", "greek", "hebrew", "cp1256", "gbk", "big5", "sjis"}
+for _, cs := range charsets {
+v, n, err := e.decodeValue(raw, mysql.MYSQL_TYPE_VAR_STRING, cs, 255)
+if err != nil {
+t.Fatalf("decodeValue error (charset=%s): %v", cs, err)
+}
+if n != int(raw[0])+1 {
+t.Fatalf("read length = %d, want %d (charset=%s)", n, int(raw[0])+1, cs)
+}
+s, ok := v.(string)
+if !ok {
+t.Fatalf("value type = %T, want string (charset=%s)", v, cs)
+}
+want := " 1rHuÎ ð XÅXÄ "
+if s != want {
+t.Fatalf("decoded string (charset=%s) = %q, want %q", cs, s, want)
+}
 }
 }