klauspost
diff --git a/‎zstd/_generate/gen.go‎
Lines changed: 80 additions & 52 deletions b/‎zstd/_generate/gen.go‎
Lines changed: 80 additions & 52 deletions
@@ -1135,9 +1135,9 @@ func (e executeSimple) executeSingleTriple(c *executeSingleTripleContext, handle
 e.copyMemoryPrecise("1", c.literals, c.outBase, ll)
 } else {
 e.copyMemoryND("1", c.literals, c.outBase, ll)
+ADDQ(ll, c.literals)
+ADDQ(ll, c.outBase)
 }
-ADDQ(ll, c.literals)
-ADDQ(ll, c.outBase)
 ADDQ(ll, c.outPosition)
 }
 
@@ -1203,7 +1203,6 @@ func (e executeSimple) executeSingleTriple(c *executeSingleTripleContext, handle
 */
 e.copyMemoryPrecise("4", ptr, c.outBase, ml)
 ADDQ(ml, c.outPosition)
-ADDQ(ml, c.outBase)
 // Note: for the current go tests this branch is taken in 99.53% cases,
 // this is why we repeat a little code here.
 handleLoop()
@@ -1219,7 +1218,6 @@ func (e executeSimple) executeSingleTriple(c *executeSingleTripleContext, handle
  }
 */
 e.copyMemoryPrecise("5", ptr, c.outBase, v)
-ADDQ(v, c.outBase)
 ADDQ(v, c.outPosition)
 SUBQ(v, ml)
 // fallback to the next block
@@ -1254,7 +1252,6 @@ func (e executeSimple) executeSingleTriple(c *executeSingleTripleContext, handle
 ADDQ(ml, c.outPosition)
 if e.safeMem {
 e.copyMemoryPrecise("2", src, c.outBase, ml)
-ADDQ(ml, c.outBase)
 } else {
 dst := GP64()
 MOVQ(c.outBase, dst)
@@ -1312,62 +1309,93 @@ func (e executeSimple) copyMemoryND(suffix string, src, dst, length reg.GPVirtua
 }
 
 // copyMemoryPrecise will copy memory in blocks of 16 bytes,
-// without overwriting nor overreading.
+// without overreading. It adds length to src and dst,
+// preserving length.
 func (e executeSimple) copyMemoryPrecise(suffix string, src, dst, length reg.GPVirtual) {
-label := "copy_" + suffix
-ofs := GP64()
-s := Mem{Base: src, Index: ofs, Scale: 1}
-d := Mem{Base: dst, Index: ofs, Scale: 1}
-
-tmp := GP64()
-XORQ(ofs, ofs)
-
-Label("copy_" + suffix + "_byte")
-TESTQ(U32(0x1), length)
-JZ(LabelRef("copy_" + suffix + "_word"))
+n := GP64()
+MOVQ(length, n)
+SUBQ(U8(16), n)
+JB(LabelRef("copy_" + suffix + "_small"))
 
-// copy one byte if length & 0x01 != 0
-MOVB(s, tmp.As8())
-MOVB(tmp.As8(), d)
-ADDQ(U8(1), ofs)
+// If length >= 16, copy blocks of 16 bytes and handle any remainder
+// by a block copy that overlaps with the last full block.
+{
+t := XMM()
 
-Label("copy_" + suffix + "_word")
-TESTQ(U32(0x2), length)
-JZ(LabelRef("copy_" + suffix + "_dword"))
+loop := "copy_" + suffix + "_loop"
+Label(loop)
+{
+MOVUPS(Mem{Base: src}, t)
+MOVUPS(t, Mem{Base: dst})
+ADDQ(U8(16), src)
+ADDQ(U8(16), dst)
+SUBQ(U8(16), n)
+JAE(LabelRef(loop))
+}
 
-// copy two bytes if length & 0x02 != 0
-MOVW(s, tmp.As16())
-MOVW(tmp.As16(), d)
-ADDQ(U8(2), ofs)
+// n is now the range [-16,-1].
+// -16 means we copy the entire last block again.
+// That should happen about 1/16th of the time,
+// so we don't bother to check for it.
+LEAQ(Mem{Base: src, Index: n, Disp: 16, Scale: 1}, src)
+LEAQ(Mem{Base: dst, Index: n, Disp: 16, Scale: 1}, dst)
+MOVUPS(Mem{Base: src, Disp: -16}, t)
+MOVUPS(t, Mem{Base: dst, Disp: -16})
 
-Label("copy_" + suffix + "_dword")
-TESTQ(U32(0x4), length)
-JZ(LabelRef("copy_" + suffix + "_qword"))
+JMP(LabelRef("copy_" + suffix + "_end"))
+}
 
-// copy four bytes if length & 0x04 != 0
-MOVL(s, tmp.As32())
-MOVL(tmp.As32(), d)
-ADDQ(U8(4), ofs)
+Label("copy_" + suffix + "_small")
+{
+ofs := GP64()
+s := Mem{Base: src, Index: ofs, Scale: 1}
+d := Mem{Base: dst, Index: ofs, Scale: 1}
 
-Label("copy_" + suffix + "_qword")
-TESTQ(U32(0x8), length)
-JZ(LabelRef("copy_" + suffix + "_test"))
+tmp := GP64()
+XORQ(ofs, ofs)
+
+Label("copy_" + suffix + "_byte")
+TESTQ(U32(0x1), length)
+JZ(LabelRef("copy_" + suffix + "_word"))
+
+// copy one byte if length & 0x01 != 0
+MOVB(s, tmp.As8())
+MOVB(tmp.As8(), d)
+ADDQ(U8(1), ofs)
+
+Label("copy_" + suffix + "_word")
+TESTQ(U32(0x2), length)
+JZ(LabelRef("copy_" + suffix + "_dword"))
+
+// copy two bytes if length & 0x02 != 0
+MOVW(s, tmp.As16())
+MOVW(tmp.As16(), d)
+ADDQ(U8(2), ofs)
+
+Label("copy_" + suffix + "_dword")
+TESTQ(U32(0x4), length)
+JZ(LabelRef("copy_" + suffix + "_qword"))
+
+// copy four bytes if length & 0x04 != 0
+MOVL(s, tmp.As32())
+MOVL(tmp.As32(), d)
+ADDQ(U8(4), ofs)
+
+Label("copy_" + suffix + "_qword")
+TESTQ(U32(0x8), length)
+JZ(LabelRef("copy_" + suffix + "_add"))
+
+// copy eight bytes if length & 0x08 != 0
+MOVQ(s, tmp)
+MOVQ(tmp, d)
+ADDQ(U8(8), ofs)
+}
 
-// copy eight bytes if length & 0x08 != 0
-MOVQ(s, tmp)
-MOVQ(tmp, d)
-ADDQ(U8(8), ofs)
-JMP(LabelRef("copy_" + suffix + "_test"))
+Label("copy_" + suffix + "_add")
+ADDQ(length, dst)
+ADDQ(length, src)
 
-// copy in 16-byte chunks
-Label(label)
-t := XMM()
-MOVUPS(s, t)
-MOVUPS(t, d)
-ADDQ(U8(16), ofs)
-Label("copy_" + suffix + "_test")
-CMPQ(ofs, length)
-JB(LabelRef(label))
+Label("copy_" + suffix + "_end")
 }
 
 // copyOverlappedMemory will copy one byte at the time from src to dst.