8290324: Move atomic operations outside of os_xxx.hpp

Reviewed-by: dholmes, kbarrett

Author: iklam

src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp

@@ -549,6 +549,10 @@ void os::current_thread_enable_wx(WXMode mode) {
  pthread_jit_write_protect_np(mode == WXExec);
 }
 
+static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
+ *(jlong *) dst = *(const jlong *) src;
+}
+
 extern "C" {
  int SpinPause() {
  return 0;
@@ -582,18 +586,19 @@ extern "C" {
  *(to--) = *(from--);
  }
  }
+
  void _Copy_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
  if (from > to) {
  const jlong *end = from + count;
  while (from < end)
- os::atomic_copy64(from++, to++);
+ atomic_copy64(from++, to++);
  }
  else if (from < to) {
  const jlong *end = from;
  from += count - 1;
  to += count - 1;
  while (from >= end)
- os::atomic_copy64(from--, to--);
+ atomic_copy64(from--, to--);
  }
  }
 

src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -35,9 +35,4 @@
  // Note: Currently only used in 64 bit Windows implementations
  static bool register_code_area(char *low, char *high) { return true; }
 
- // Atomically copy 64 bits of data
- static void atomic_copy64(const volatile void *src, volatile void *dst) {
- *(jlong *) dst = *(const jlong *) src;
- }
-
 #endif // OS_CPU_BSD_AARCH64_OS_BSD_AARCH64_HPP

src/hotspot/os_cpu/bsd_zero/atomic_bsd_zero.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2007, 2008, 2011, 2015, Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -285,12 +285,31 @@ inline T Atomic::PlatformCmpxchg<8>::operator()(T volatile* dest,
  return value;
 }
 
+// Atomically copy 64 bits of data
+static void atomic_copy64(const volatile void *src, volatile void *dst) {
+#if defined(PPC32)
+ double tmp;
+ asm volatile ("lfd %0, 0(%1)\n"
+ "stfd %0, 0(%2)\n"
+ : "=f"(tmp)
+ : "b"(src), "b"(dst));
+#elif defined(S390) && !defined(_LP64)
+ double tmp;
+ asm volatile ("ld %0, 0(%1)\n"
+ "std %0, 0(%2)\n"
+ : "=r"(tmp)
+ : "a"(src), "a"(dst));
+#else
+ *(jlong *) dst = *(const jlong *) src;
+#endif
+}
+
 template<>
 template<typename T>
 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
  STATIC_ASSERT(8 == sizeof(T));
  volatile int64_t dest;
- os::atomic_copy64(reinterpret_cast<const volatile int64_t*>(src), reinterpret_cast<volatile int64_t*>(&dest));
+ atomic_copy64(reinterpret_cast<const volatile int64_t*>(src), reinterpret_cast<volatile int64_t*>(&dest));
  return PrimitiveConversions::cast<T>(dest);
 }
 
@@ -299,7 +318,7 @@ template<typename T>
 inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
  T store_value) const {
  STATIC_ASSERT(8 == sizeof(T));
- os::atomic_copy64(reinterpret_cast<const volatile int64_t*>(&store_value), reinterpret_cast<volatile int64_t*>(dest));
+ atomic_copy64(reinterpret_cast<const volatile int64_t*>(&store_value), reinterpret_cast<volatile int64_t*>(dest));
 }
 
 #endif // OS_CPU_BSD_ZERO_ATOMIC_BSD_ZERO_HPP

src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp

@@ -31,6 +31,7 @@
 // no precompiled headers
 #include "jvm.h"
 #include "asm/assembler.inline.hpp"
+#include "atomic_bsd_zero.hpp"
 #include "classfile/vmSymbols.hpp"
 #include "code/icBuffer.hpp"
 #include "code/vtableStubs.hpp"
@@ -295,14 +296,14 @@ extern "C" {
  if (from > to) {
  const jlong *end = from + count;
  while (from < end)
- os::atomic_copy64(from++, to++);
+ atomic_copy64(from++, to++);
  }
  else if (from < to) {
  const jlong *end = from;
  from += count - 1;
  to += count - 1;
  while (from >= end)
- os::atomic_copy64(from--, to--);
+ atomic_copy64(from--, to--);
  }
  }
 

src/hotspot/os_cpu/bsd_zero/os_bsd_zero.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2007, 2008, 2010 Red Hat, Inc.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -32,23 +32,4 @@
  // Note: Currently only used in 64 bit Windows implementations
  static bool register_code_area(char *low, char *high) { return true; }
 
- // Atomically copy 64 bits of data
- static void atomic_copy64(const volatile void *src, volatile void *dst) {
-#if defined(PPC32)
- double tmp;
- asm volatile ("lfd %0, 0(%1)\n"
- "stfd %0, 0(%2)\n"
- : "=f"(tmp)
- : "b"(src), "b"(dst));
-#elif defined(S390) && !defined(_LP64)
- double tmp;
- asm volatile ("ld %0, 0(%1)\n"
- "std %0, 0(%2)\n"
- : "=r"(tmp)
- : "a"(src), "a"(dst));
-#else
- *(jlong *) dst = *(const jlong *) src;
-#endif
- }
-
 #endif // OS_CPU_BSD_ZERO_OS_BSD_ZERO_HPP

src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp

@@ -387,6 +387,10 @@ int os::extra_bang_size_in_bytes() {
  return 0;
 }
 
+static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
+ *(jlong *) dst = *(const jlong *) src;
+}
+
 extern "C" {
  int SpinPause() {
  using spin_wait_func_ptr_t = void (*)();
@@ -433,18 +437,19 @@ extern "C" {
  *(to--) = *(from--);
  }
  }
+
  void _Copy_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
  if (from > to) {
  const jlong *end = from + count;
  while (from < end)
- os::atomic_copy64(from++, to++);
+ atomic_copy64(from++, to++);
  }
  else if (from < to) {
  const jlong *end = from;
  from += count - 1;
  to += count - 1;
  while (from >= end)
- os::atomic_copy64(from--, to--);
+ atomic_copy64(from--, to--);
  }
  }
 

src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -36,9 +36,4 @@
  // Note: Currently only used in 64 bit Windows implementations
  static bool register_code_area(char *low, char *high) { return true; }
 
- // Atomically copy 64 bits of data
- static void atomic_copy64(const volatile void *src, volatile void *dst) {
- *(jlong *) dst = *(const jlong *) src;
- }
-
 #endif // OS_CPU_LINUX_AARCH64_OS_LINUX_AARCH64_HPP

src/hotspot/os_cpu/linux_arm/atomic_linux_arm.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,43 @@
 #ifndef OS_CPU_LINUX_ARM_ATOMIC_LINUX_ARM_HPP
 #define OS_CPU_LINUX_ARM_ATOMIC_LINUX_ARM_HPP
 
+#include "memory/allStatic.hpp"
 #include "runtime/os.hpp"
 #include "runtime/vm_version.hpp"
 
 // Implementation of class atomic
 
+class ARMAtomicFuncs : AllStatic {
+public:
+ typedef int64_t (*cmpxchg_long_func_t)(int64_t, int64_t, volatile int64_t*);
+ typedef int64_t (*load_long_func_t)(const volatile int64_t*);
+ typedef void (*store_long_func_t)(int64_t, volatile int64_t*);
+ typedef int32_t (*atomic_add_func_t)(int32_t add_value, volatile int32_t *dest);
+ typedef int32_t (*atomic_xchg_func_t)(int32_t exchange_value, volatile int32_t *dest);
+ typedef int32_t (*cmpxchg_func_t)(int32_t, int32_t, volatile int32_t*);
+
+ static cmpxchg_long_func_t _cmpxchg_long_func;
+ static load_long_func_t _load_long_func;
+ static store_long_func_t _store_long_func;
+ static atomic_add_func_t _add_func;
+ static atomic_xchg_func_t _xchg_func;
+ static cmpxchg_func_t _cmpxchg_func;
+
+ static int64_t cmpxchg_long_bootstrap(int64_t, int64_t, volatile int64_t*);
+
+ static int64_t load_long_bootstrap(const volatile int64_t*);
+
+ static void store_long_bootstrap(int64_t, volatile int64_t*);
+
+ static int32_t add_bootstrap(int32_t add_value, volatile int32_t *dest);
+
+ static int32_t xchg_bootstrap(int32_t exchange_value, volatile int32_t *dest);
+
+ static int32_t cmpxchg_bootstrap(int32_t compare_value,
+ int32_t exchange_value,
+ volatile int32_t *dest);
+};
+
 /*
  * Atomic long operations on 32-bit ARM
  * ARM v7 supports LDREXD/STREXD synchronization instructions so no problem.
@@ -49,15 +81,15 @@ template<typename T>
 inline T Atomic::PlatformLoad<8>::operator()(T const volatile* src) const {
  STATIC_ASSERT(8 == sizeof(T));
  return PrimitiveConversions::cast<T>(
- (*os::atomic_load_long_func)(reinterpret_cast<const volatile int64_t*>(src)));
+ (*ARMAtomicFuncs::_load_long_func)(reinterpret_cast<const volatile int64_t*>(src)));
 }
 
 template<>
 template<typename T>
 inline void Atomic::PlatformStore<8>::operator()(T volatile* dest,
  T store_value) const {
  STATIC_ASSERT(8 == sizeof(T));
- (*os::atomic_store_long_func)(
+ (*ARMAtomicFuncs::_store_long_func)(
  PrimitiveConversions::cast<int64_t>(store_value), reinterpret_cast<volatile int64_t*>(dest));
 }
 
@@ -83,7 +115,7 @@ inline D Atomic::PlatformAdd<4>::add_and_fetch(D volatile* dest, I add_value,
  atomic_memory_order order) const {
  STATIC_ASSERT(4 == sizeof(I));
  STATIC_ASSERT(4 == sizeof(D));
- return add_using_helper<int32_t>(os::atomic_add_func, dest, add_value);
+ return add_using_helper<int32_t>(ARMAtomicFuncs::_add_func, dest, add_value);
 }
 
 
@@ -93,7 +125,7 @@ inline T Atomic::PlatformXchg<4>::operator()(T volatile* dest,
  T exchange_value,
  atomic_memory_order order) const {
  STATIC_ASSERT(4 == sizeof(T));
- return xchg_using_helper<int32_t>(os::atomic_xchg_func, dest, exchange_value);
+ return xchg_using_helper<int32_t>(ARMAtomicFuncs::_xchg_func, dest, exchange_value);
 }
 
 
@@ -108,15 +140,15 @@ inline int32_t reorder_cmpxchg_func(int32_t exchange_value,
  int32_t volatile* dest,
  int32_t compare_value) {
  // Warning: Arguments are swapped to avoid moving them for kernel call
- return (*os::atomic_cmpxchg_func)(compare_value, exchange_value, dest);
+ return (*ARMAtomicFuncs::_cmpxchg_func)(compare_value, exchange_value, dest);
 }
 
 inline int64_t reorder_cmpxchg_long_func(int64_t exchange_value,
  int64_t volatile* dest,
  int64_t compare_value) {
  assert(VM_Version::supports_cx8(), "Atomic compare and exchange int64_t not supported on this architecture!");
  // Warning: Arguments are swapped to avoid moving them for kernel call
- return (*os::atomic_cmpxchg_long_func)(compare_value, exchange_value, dest);
+ return (*ARMAtomicFuncs::_cmpxchg_long_func)(compare_value, exchange_value, dest);
 }