From 476ec6be3f75c70c50bd1552c624abca098ddba2 Mon Sep 17 00:00:00 2001
Date: Wed, 18 Mar 2020 10:25:06 +0000
Subject: [PATCH] 8209375: ZGC: Use dynamic base address for mark stack space

Summary: <gc>: <mark stack needs atomic change>
LLT: jdk11u/test/hotspot/jtreg/vmTestbase/gc/gctests/SoftReference/soft004/soft004.java
Bug url: https://bugs.openjdk.java.net/browse/JDK-8209375
---
 src/hotspot/share/gc/z/zGlobals.hpp   |  7 +---
 src/hotspot/share/gc/z/zMarkStack.cpp | 74 +++++++++++++++--------------------
 src/hotspot/share/gc/z/zMarkStack.hpp |  1 +
 src/hotspot/share/gc/z/z_globals.hpp  |  6 +--
 4 files changed, 38 insertions(+), 50 deletions(-)

diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp
index 080ea5c0e..0f9e9dcb4 100644
--- a/src/hotspot/share/gc/z/zGlobals.hpp
+++ b/src/hotspot/share/gc/z/zGlobals.hpp
@@ -117,11 +117,8 @@ extern uintptr_t  ZAddressWeakBadMask;
 // Marked state
 extern uintptr_t  ZAddressMetadataMarked;
 
-// Address space for mark stack allocations
-const size_t      ZMarkStackSpaceSizeShift      = 40; // 1TB
-const size_t      ZMarkStackSpaceSize           = (size_t)1 << ZMarkStackSpaceSizeShift;
-const uintptr_t   ZMarkStackSpaceStart          = ZAddressSpaceEnd + ZMarkStackSpaceSize;
-const uintptr_t   ZMarkStackSpaceEnd            = ZMarkStackSpaceStart + ZMarkStackSpaceSize;
+// Mark stack space
+extern uintptr_t  ZMarkStackSpaceStart;
 const size_t      ZMarkStackSpaceExpandSize     = (size_t)1 << 25; // 32M
 
 // Mark stack and magazine sizes
diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp
index 52fe51ece..9cc768956 100644
--- a/src/hotspot/share/gc/z/zMarkStack.cpp
+++ b/src/hotspot/share/gc/z/zMarkStack.cpp
@@ -28,58 +28,44 @@
 #include "gc/z/zMarkStack.inline.hpp"
 #include "logging/log.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
 #include "utilities/debug.hpp"
 
-#include <sys/mman.h>
-#include <sys/types.h>
+uintptr_t ZMarkStackSpaceStart;
 
 ZMarkStackSpace::ZMarkStackSpace() :
     _expand_lock(),
+    _start(0),
     _top(0),
     _end(0) {
-  assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small");
-  assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large");
-
+  assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
   // Reserve address space
-  const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize,
-                         PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
-  if (res != (void*)ZMarkStackSpaceStart) {
-    log_error(gc, marking)("Failed to reserve address space for marking stacks");
+  const size_t size = ZMarkStackSpaceLimit;
+  const size_t alignment = (size_t)os::vm_allocation_granularity();
+  const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
+  if (addr == 0) {
+    log_error(gc, marking)("Failed to reserve address space for mark stacks");
     return;
   }
 
   // Successfully initialized
-  _top = _end = ZMarkStackSpaceStart;
-}
+  _start = _top = _end = addr;
 
-bool ZMarkStackSpace::is_initialized() const {
-  return _top != 0;
+  // Register mark stack space start
+  ZMarkStackSpaceStart = _start;
 }
 
-bool ZMarkStackSpace::expand() {
-  const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax;
-  if (_end + ZMarkStackSpaceExpandSize > max) {
-    // Expansion limit reached
-    return false;
-  }
-
-  void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize,
-                         PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0);
-  if (res == MAP_FAILED) {
-    ZErrno err;
-    log_error(gc, marking)("Failed to map memory for marking stacks (%s)", err.to_string());
-    return false;
-  }
-
-  return true;
+bool ZMarkStackSpace::is_initialized() const {
+  return _start != 0;
 }
 
 uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
-  uintptr_t top = _top;
+  uintptr_t top = Atomic::load(&_top);
 
   for (;;) {
+    const uintptr_t end = Atomic::load(&_end);
     const uintptr_t new_top = top + size;
-    if (new_top > _end) {
+    if (new_top > end) {
       // Not enough space left
       return 0;
     }
@@ -104,24 +90,28 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
     return addr;
   }
 
-  // Expand stack space
-  if (!expand()) {
-    // We currently can't handle the situation where we
-    // are running out of mark stack space.
-    fatal("Mark stack overflow (allocated " SIZE_FORMAT "M, size " SIZE_FORMAT "M, max " SIZE_FORMAT "M),"
-          " use -XX:ZMarkStacksMax=? to increase this limit",
-          (_end - ZMarkStackSpaceStart) / M, size / M, ZMarkStacksMax / M);
-    return 0;
+  // Check expansion limit
+  const size_t expand_size = ZMarkStackSpaceExpandSize;
+  const size_t old_size = _end - _start;
+  const size_t new_size = old_size + expand_size;
+  if (new_size > ZMarkStackSpaceLimit) {
+    // Expansion limit reached. This is a fatal error since we
+    // currently can't recover from running out of mark stack space.
+    fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
+          "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
+          ZMarkStackSpaceLimit / M);
   }
 
   log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
-                         (_end - ZMarkStackSpaceStart) / M,
-                         (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M);
+                         old_size / M, new_size / M);
+
+  // Expand
+  os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
 
   // Increment top before end to make sure another
   // thread can't steal out newly expanded space.
   addr = Atomic::add(size, &_top) - size;
-  _end += ZMarkStackSpaceExpandSize;
+  Atomic::add(expand_size, &_end);
 
   return addr;
 }
diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp
index b68b9faa3..12f3e4eca 100644
--- a/src/hotspot/share/gc/z/zMarkStack.hpp
+++ b/src/hotspot/share/gc/z/zMarkStack.hpp
@@ -76,6 +76,7 @@ typedef ZStackList<ZMarkStackMagazine>               ZMarkStackMagazineList;
 class ZMarkStackSpace {
 private:
   ZLock              _expand_lock;
+  uintptr_t          _start;
   volatile uintptr_t _top;
   volatile uintptr_t _end;
 
diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp
index 9e0f8985b..8cee59be7 100644
--- a/src/hotspot/share/gc/z/z_globals.hpp
+++ b/src/hotspot/share/gc/z/z_globals.hpp
@@ -53,9 +53,9 @@
           "Allow Java threads to stall and wait for GC to complete "        \
           "instead of immediately throwing an OutOfMemoryError")            \
                                                                             \
-  product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G),           \
-          "Maximum number of bytes allocated for marking stacks")           \
-          range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G))                    \
+  product(size_t, ZMarkStackSpaceLimit, 8*G,                                \
+          "Maximum number of bytes allocated for mark stacks")              \
+          range(32*M, 1024*G)                                               \
                                                                             \
   product(uint, ZCollectionInterval, 0,                                     \
           "Force GC at a fixed time interval (in seconds)")                 \
-- 
2.12.3