Refactor barrier interface (#177)

* Add arraycopy barrier; remove clone barrier; barrier elision

* wip

* object_reference_write

* WIP array_copy

* Update array copy barrier

* Update array copy barrier

* wip C1 barrier

* Fix C1 barrier

* refactor

* pre barrier slow path

* minor

* add `enable_oop_arraycopy_prologue`

* Fix

* Cleanup

* cleanup

* switch to post-barrier

* cleanup

* WIP: Move more shared methods out of object barrier

* Move C1 common code to MMTkBarrierSetC1

* cleanup

* minor

* minor

* Update

* Add comments

* Update Cargo.toml

* Update

* Update Cargo.lock

* rename `array_copy` -> `memory_region_copy`

* Update

* Update mmtk-core

* Update Cargo.lock

* minor

* Add newline

* Rename

* Update mmtk-core

Author: wenyuzhao

mmtk/Cargo.lock

@@ -17,7 +17,7 @@ lto = true
 [package.metadata.openjdk]
 # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works.
 openjdk_repo = "https://github.com/mmtk/openjdk.git"
-openjdk_version = "ad1809129d6288e7f8226990840e5d32aab0ea34"
+openjdk_version = "c82e5c44adced4383162826c2c3933a83cfb139b"
 
 [dependencies]
 libc = "0.2"
@@ -29,7 +29,7 @@ once_cell = "1.10.0"
 # - change branch
 # - change repo name
 # But other changes including adding/removing whitespaces in commented lines may break the CI.
-mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "4ae23b1dc0e14f62c196c12c93818e00e9de7cc5" }
+mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "9473d7b4b92747d3c0998af3c41a6ef50c9c94e3" }
 # Uncomment the following to build locally
 # mmtk = { path = "../repos/mmtk-core" }
 

mmtk/Cargo.toml

@@ -9,6 +9,7 @@ use mmtk::plan::BarrierSelector;
 use mmtk::scheduler::GCController;
 use mmtk::scheduler::GCWorker;
 use mmtk::util::alloc::AllocatorSelector;
+use mmtk::util::constants::LOG_BYTES_IN_ADDRESS;
 use mmtk::util::opaque_pointer::*;
 use mmtk::util::{Address, ObjectReference};
 use mmtk::AllocationSemantics;
@@ -308,21 +309,71 @@ pub extern "C" fn executable() -> bool {
     true
 }
 
+/// Full pre barrier
 #[no_mangle]
-pub extern "C" fn post_write_barrier(mutator: &'static mut Mutator<OpenJDK>, obj: ObjectReference) {
+pub extern "C" fn mmtk_object_reference_write_pre(
+    mutator: &'static mut Mutator<OpenJDK>,
+    src: ObjectReference,
+    slot: Address,
+    target: ObjectReference,
+) {
+    mutator
+        .barrier()
+        .object_reference_write_pre(src, slot, target);
+}
+
+/// Full post barrier
+#[no_mangle]
+pub extern "C" fn mmtk_object_reference_write_post(
+    mutator: &'static mut Mutator<OpenJDK>,
+    src: ObjectReference,
+    slot: Address,
+    target: ObjectReference,
+) {
+    mutator
+        .barrier()
+        .object_reference_write_post(src, slot, target);
+}
+
+/// Barrier slow-path call
+#[no_mangle]
+pub extern "C" fn mmtk_object_reference_write_slow(
+    mutator: &'static mut Mutator<OpenJDK>,
+    src: ObjectReference,
+    slot: Address,
+    target: ObjectReference,
+) {
+    mutator
+        .barrier()
+        .object_reference_write_slow(src, slot, target);
+}
+
+/// Array-copy pre-barrier
+#[no_mangle]
+pub extern "C" fn mmtk_array_copy_pre(
+    mutator: &'static mut Mutator<OpenJDK>,
+    src: Address,
+    dst: Address,
+    count: usize,
+) {
+    let bytes = count << LOG_BYTES_IN_ADDRESS;
     mutator
         .barrier()
-        .post_write_barrier(mmtk::plan::BarrierWriteTarget::Object(obj))
+        .memory_region_copy_pre(src..src + bytes, dst..dst + bytes);
 }
 
+/// Array-copy post-barrier
 #[no_mangle]
-pub extern "C" fn post_write_barrier_slow(
+pub extern "C" fn mmtk_array_copy_post(
     mutator: &'static mut Mutator<OpenJDK>,
-    obj: ObjectReference,
+    src: Address,
+    dst: Address,
+    count: usize,
 ) {
+    let bytes = count << LOG_BYTES_IN_ADDRESS;
     mutator
         .barrier()
-        .post_write_barrier_slow(mmtk::plan::BarrierWriteTarget::Object(obj))
+        .memory_region_copy_post(src..src + bytes, dst..dst + bytes);
 }
 
 // finalization

mmtk/src/api.rs

@@ -5,6 +5,7 @@ extern crate lazy_static;
 extern crate once_cell;
 
 use std::collections::HashMap;
+use std::ops::Range;
 use std::ptr::null_mut;
 use std::sync::atomic::AtomicUsize;
 use std::sync::Mutex;
@@ -129,6 +130,7 @@ impl VMBinding for OpenJDK {
     type VMReferenceGlue = reference_glue::VMReferenceGlue;
 
     type VMEdge = OpenJDKEdge;
+    type VMMemorySlice = Range<Address>;
 }
 
 use std::sync::atomic::AtomicBool;

mmtk/src/lib.rs

@@ -2,51 +2,31 @@
 #include "mmtkObjectBarrier.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 
-void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* obj) {
-  ::post_write_barrier_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) obj);
-}
-
-void MMTkObjectBarrierSetRuntime::record_modified_node_full(void* obj) {
-  ::post_write_barrier((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) obj);
-}
-
-void MMTkObjectBarrierSetRuntime::record_modified_node(oop src) {
-#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH
+void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const {
+#if MMTK_ENABLE_BARRIER_FASTPATH
   intptr_t addr = (intptr_t) (void*) src;
   uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6));
   intptr_t shift = (addr >> 3) & 0b111;
   uint8_t byte_val = *meta_addr;
   if (((byte_val >> shift) & 1) == 1) {
-    record_modified_node_slow((void*) src);
+    // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src);
+    object_reference_write_slow_call((void*) src, (void*) slot, (void*) target);
   }
 #else
-  record_modified_node_full((void*) src);
+  object_reference_write_post_call((void*) src, (void*) slot, (void*) target);
 #endif
 }
 
 #define __ masm->
 
-void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) {
-  bool in_heap = (decorators & IN_HEAP) != 0;
-  bool as_normal = (decorators & AS_NORMAL) != 0;
-  assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported");
-
-  if (!in_heap || val == noreg) {
-    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
-    return;
-  }
-
-  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
-
-  record_modified_node(masm, dst.base(), tmp1, tmp2);
-}
-
-void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
-#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH
+void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {
+  if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return;
+#if MMTK_ENABLE_BARRIER_FASTPATH
   Label done;
 
   Register tmp3 = rscratch1;
   Register tmp4 = rscratch2;
+  Register obj = dst.base();
   assert_different_registers(obj, tmp2, tmp3);
   assert_different_registers(tmp4, rcx);
 
@@ -69,18 +49,32 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, R
   __ cmpptr(tmp2, 1);
   __ jcc(Assembler::notEqual, done);
 
-  assert_different_registers(c_rarg0, obj);
   __ movptr(c_rarg0, obj);
-  __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 1);
+  __ lea(c_rarg1, dst);
+  __ movptr(c_rarg2, val == noreg ?  (int32_t) NULL_WORD : val);
+  __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3);
 
   __ bind(done);
 #else
-  assert_different_registers(c_rarg0, obj);
   __ movptr(c_rarg0, obj);
-  __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_full), 1);
+  __ lea(c_rarg1, dst);
+  __ movptr(c_rarg2, val == noreg ?  (int32_t) NULL_WORD : val);
+  __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3);
 #endif
 }
 
+void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) {
+  const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+  if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) {
+    __ pusha();
+    __ movptr(c_rarg0, src);
+    __ movptr(c_rarg1, dst);
+    __ movptr(c_rarg2, count);
+    __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), 3);
+    __ popa();
+  }
+}
+
 #undef __
 
 #ifdef ASSERT
@@ -89,7 +83,7 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, R
 #define __ gen->lir()->
 #endif
 
-void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) {
+void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {
   LIRGenerator* gen = access.gen();
   DecoratorSet decorators = access.decorators();
   if ((decorators & IN_HEAP) == 0) return;
@@ -123,35 +117,31 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src
     new_val = new_val_reg;
   }
   assert(new_val->is_register(), "must be a register at this point");
-  CodeStub* slow = new MMTkObjectBarrierStub(src, slot, new_val);
+  CodeStub* slow = new MMTkC1BarrierStub(src, slot, new_val);
 
-#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH
+#if MMTK_ENABLE_BARRIER_FASTPATH
   LIR_Opr addr = src;
   // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6));
   LIR_Opr offset = gen->new_pointer_register();
   __ move(addr, offset);
-  __ shift_right(offset, 6, offset);
+  __ unsigned_shift_right(offset, 6, offset);
   LIR_Opr base = gen->new_pointer_register();
   __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base);
   LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE);
-  // intptr_t shift = (addr >> 3) & 0b111;
-  LIR_Opr shift_long = gen->new_pointer_register();
-  __ move(addr, shift_long);
-  __ shift_right(shift_long, 3, shift_long);
-  __ logical_and(shift_long, LIR_OprFact::longConst(0b111), shift_long);
-  LIR_Opr shift_int = gen->new_register(T_INT);
-  __ convert(Bytecodes::_l2i, shift_long, shift_int);
-  LIR_Opr shift = LIRGenerator::shiftCountOpr();
-  __ move(shift_int, shift);
   // uint8_t byte_val = *meta_addr;
   LIR_Opr byte_val = gen->new_register(T_INT);
   __ move(meta_addr, byte_val);
+  // intptr_t shift = (addr >> 3) & 0b111;
+  LIR_Opr shift = gen->new_register(T_INT);
+  __ move(addr, shift);
+  __ unsigned_shift_right(shift, 3, shift);
+  __ logical_and(shift, LIR_OprFact::intConst(0b111), shift);
   // if (((byte_val >> shift) & 1) == 1) slow;
   LIR_Opr result = byte_val;
-  __ shift_right(result, shift, result, LIR_OprFact::illegalOpr);
+  __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr);
   __ logical_and(result, LIR_OprFact::intConst(1), result);
   __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1));
-  __ branch(lir_cond_equal, LP64_ONLY(T_LONG) NOT_LP64(T_INT), slow);
+  __ branch(lir_cond_equal, T_BYTE, slow);
 #else
   __ jump(slow);
 #endif
@@ -163,24 +153,12 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src
 
 #define __ ideal.
 
-const TypeFunc* record_modified_node_entry_Type() {
-  const Type **fields = TypeTuple::fields(1);
-  fields[TypeFunc::Parms+0] = TypeOopPtr::BOTTOM; // oop src
-  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
-  fields = TypeTuple::fields(0);
-  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
-  return TypeFunc::make(domain, range);
-}
-
-void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* val) const {
-  if (val != NULL && val->is_Con()) {
-    const Type* t = val->bottom_type();
-    if (t == TypePtr::NULL_PTR) return;
-  }
+void MMTkObjectBarrierSetC2::object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const {
+  if (can_remove_barrier(kit, &kit->gvn(), src, slot, val, /* skip_const_null */ true)) return;
 
   MMTkIdealKit ideal(kit, true);
 
-#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH
+#if MMTK_ENABLE_BARRIER_FASTPATH
   Node* no_base = __ top();
   float unlikely  = PROB_UNLIKELY(0.999);
 
@@ -193,12 +171,12 @@ void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node
   Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1));
 
   __ if_then(result, BoolTest::ne, zero, unlikely); {
-    const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM);
-    Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src);
+    const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM);
+    Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val);
   } __ end_if();
 #else
-  const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM);
-  Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_full), "record_modified_node", src);
+  const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM);
+  Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), "mmtk_barrier_call", src, slot, val);
 #endif
 
   kit->final_sync(ideal); // Final sync IdealKit and GraphKit.