Add different misaligned path for archs with unaligned support

Author: nbdd0121

build.rs

@@ -33,6 +33,11 @@ fn main() {
  println!("cargo:rustc-cfg=feature=\"mem\"");
  }
 
+ // These targets have hardware unaligned access support.
+ if target.contains("x86_64") || target.contains("i686") || target.contains("aarch64") {
+ println!("cargo:rustc-cfg=feature=\"mem-unaligned\"");
+ }
+
  // NOTE we are going to assume that llvm-target, what determines our codegen option, matches the
  // target triple. This is usually correct for our built-in targets but can break in presence of
  // custom targets, which can have arbitrary names.

src/mem/impls.rs

@@ -16,6 +16,14 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 {
  16
 };
 
+#[cfg(feature = "mem-unaligned")]
+unsafe fn read_usize_unaligned(x: *const usize) -> usize {
+ // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which
+ // is translated to memcpy in LLVM.
+ let x_read = (x as *const [u8; core::mem::size_of::<usize>()]).read();
+ core::mem::transmute(x_read)
+}
+
 #[inline(always)]
 pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) {
  #[inline(always)]
@@ -41,6 +49,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
  }
  }
 
+ #[cfg(not(feature = "mem-unaligned"))]
  #[inline(always)]
  unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
  let mut dest_usize = dest as *mut usize;
@@ -69,6 +78,20 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
  }
  }
 
+ #[cfg(feature = "mem-unaligned")]
+ #[inline(always)]
+ unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
+ let mut dest_usize = dest as *mut usize;
+ let mut src_usize = src as *mut usize;
+ let dest_end = dest.add(n) as *mut usize;
+
+ while dest_usize < dest_end {
+ *dest_usize = read_usize_unaligned(src_usize);
+ dest_usize = dest_usize.add(1);
+ src_usize = src_usize.add(1);
+ }
+ }
+
  if n >= WORD_COPY_THRESHOLD {
  // Align dest
  // Because of n >= 2 * WORD_SIZE, dst_misalignment < n
@@ -119,6 +142,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
  }
  }
 
+ #[cfg(not(feature = "mem-unaligned"))]
  #[inline(always)]
  unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
  let mut dest_usize = dest as *mut usize;
@@ -147,6 +171,20 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
  }
  }
 
+ #[cfg(feature = "mem-unaligned")]
+ #[inline(always)]
+ unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
+ let mut dest_usize = dest as *mut usize;
+ let mut src_usize = src as *mut usize;
+ let dest_start = dest.sub(n) as *mut usize;
+
+ while dest_start < dest_usize {
+ dest_usize = dest_usize.sub(1);
+ src_usize = src_usize.sub(1);
+ *dest_usize = read_usize_unaligned(src_usize as *const u8);
+ }
+ }
+
  let mut dest = dest.add(n);
  let mut src = src.add(n);