6890673: Eliminate allocations immediately after EA

Try to eliminate allocations and related locks immediately after escape analysis.

Reviewed-by: never

hotspot/src/share/vm/opto/block.cpp

@@ -898,45 +898,41 @@ void PhaseCFG::dump_headers() {
 void PhaseCFG::verify( ) const {
 #ifdef ASSERT
   // Verify sane CFG
-  for( uint i = 0; i < _num_blocks; i++ ) {
+  for (uint i = 0; i < _num_blocks; i++) {
     Block *b = _blocks[i];
     uint cnt = b->_nodes.size();
     uint j;
-    for( j = 0; j < cnt; j++ ) {
+    for (j = 0; j < cnt; j++)  {
       Node *n = b->_nodes[j];
       assert( _bbs[n->_idx] == b, "" );
-      if( j >= 1 && n->is_Mach() &&
+      if (j >= 1 && n->is_Mach() &&
-          n->as_Mach()->ideal_Opcode() == Op_CreateEx ) {
+          n->as_Mach()->ideal_Opcode() == Op_CreateEx) {
-        assert( j == 1 || b->_nodes[j-1]->is_Phi(),
+        assert(j == 1 || b->_nodes[j-1]->is_Phi(),
-                "CreateEx must be first instruction in block" );
+               "CreateEx must be first instruction in block");
       }
-      for( uint k = 0; k < n->req(); k++ ) {
+      for (uint k = 0; k < n->req(); k++) {
         Node *def = n->in(k);
-        if( def && def != n ) {
+        if (def && def != n) {
-          assert( _bbs[def->_idx] || def->is_Con(),
+          assert(_bbs[def->_idx] || def->is_Con(),
-                  "must have block; constants for debug info ok" );
+                 "must have block; constants for debug info ok");
           // Verify that instructions in the block is in correct order.
           // Uses must follow their definition if they are at the same block.
           // Mostly done to check that MachSpillCopy nodes are placed correctly
           // when CreateEx node is moved in build_ifg_physical().
-          if( _bbs[def->_idx] == b &&
+          if (_bbs[def->_idx] == b &&
               !(b->head()->is_Loop() && n->is_Phi()) &&
               // See (+++) comment in reg_split.cpp
-              !(n->jvms() != NULL && n->jvms()->is_monitor_use(k)) ) {
+              !(n->jvms() != NULL && n->jvms()->is_monitor_use(k))) {
             bool is_loop = false;
             if (n->is_Phi()) {
-              for( uint l = 1; l < def->req(); l++ ) {
+              for (uint l = 1; l < def->req(); l++) {
                 if (n == def->in(l)) {
                   is_loop = true;
                   break; // Some kind of loop
                 }
               }
             }
-            assert( is_loop || b->find_node(def) < j, "uses must follow definitions" );
+            assert(is_loop || b->find_node(def) < j, "uses must follow definitions");
-          }
-          if( def->is_SafePointScalarObject() ) {
-            assert(_bbs[def->_idx] == b, "SafePointScalarObject Node should be at the same block as its SafePoint node");
-            assert(_bbs[def->_idx] == _bbs[def->in(0)->_idx], "SafePointScalarObject Node should be at the same block as its control edge");
           }
         }
       }
@@ -946,12 +942,11 @@ void PhaseCFG::verify( ) const {
     Node *bp = (Node*)b->_nodes[b->_nodes.size()-1]->is_block_proj();
     assert( bp, "last instruction must be a block proj" );
     assert( bp == b->_nodes[j], "wrong number of successors for this block" );
-    if( bp->is_Catch() ) {
+    if (bp->is_Catch()) {
-      while( b->_nodes[--j]->is_MachProj() ) ;
+      while (b->_nodes[--j]->is_MachProj()) ;
-      assert( b->_nodes[j]->is_MachCall(), "CatchProj must follow call" );
+      assert(b->_nodes[j]->is_MachCall(), "CatchProj must follow call");
-    }
+    } else if (bp->is_Mach() && bp->as_Mach()->ideal_Opcode() == Op_If) {
-    else if( bp->is_Mach() && bp->as_Mach()->ideal_Opcode() == Op_If ) {
+      assert(b->_num_succs == 2, "Conditional branch must have two targets");
-      assert( b->_num_succs == 2, "Conditional branch must have two targets");
     }
   }
 #endif

hotspot/src/share/vm/opto/callnode.cpp

@@ -1071,8 +1071,11 @@ SafePointScalarObjectNode::SafePointScalarObjectNode(const TypeOopPtr* tp,
   init_class_id(Class_SafePointScalarObject);
 }
 
-bool SafePointScalarObjectNode::pinned() const { return true; }
+// Do not allow value-numbering for SafePointScalarObject node.
-bool SafePointScalarObjectNode::depends_only_on_test() const { return false; }
+uint SafePointScalarObjectNode::hash() const { return NO_HASH; }
+uint SafePointScalarObjectNode::cmp( const Node &n ) const {
+  return (&n == this); // Always fail except on self
+}
 
 uint SafePointScalarObjectNode::ideal_reg() const {
   return 0; // No matching to machine instruction
@@ -1096,7 +1099,6 @@ SafePointScalarObjectNode::clone(int jvms_adj, Dict* sosn_map) const {
   if (cached != NULL) {
     return (SafePointScalarObjectNode*)cached;
   }
-  Compile* C = Compile::current();
   SafePointScalarObjectNode* res = (SafePointScalarObjectNode*)Node::clone();
   res->_first_index += jvms_adj;
   sosn_map->Insert((void*)this, (void*)res);
@@ -1142,6 +1144,8 @@ uint AllocateArrayNode::size_of() const { return sizeof(*this); }
 
 Node* AllocateArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
 
   const Type* type = phase->type(Ideal_length());
   if (type->isa_int() && type->is_int()->_hi < 0) {
@@ -1522,13 +1526,16 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 
   // perform any generic optimizations first (returns 'this' or NULL)
   Node *result = SafePointNode::Ideal(phase, can_reshape);
+  if (result != NULL)  return result;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
 
   // Now see if we can optimize away this lock.  We don't actually
   // remove the locking here, we simply set the _eliminate flag which
   // prevents macro expansion from expanding the lock.  Since we don't
   // modify the graph, the value returned from this function is the
   // one computed above.
-  if (result == NULL && can_reshape && EliminateLocks && !is_eliminated()) {
+  if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) {
     //
     // If we are locking an unescaped object, the lock/unlock is unnecessary
     //
@@ -1537,8 +1544,16 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     if (cgr != NULL)
       es = cgr->escape_state(obj_node());
     if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) {
-      // Mark it eliminated to update any counters
+      if (!is_eliminated()) {
-      this->set_eliminated();
+        // Mark it eliminated to update any counters
+        this->set_eliminated();
+      } else {
+        assert(is_coarsened(), "sanity");
+        // The lock could be marked eliminated by lock coarsening
+        // code during first IGVN before EA. Clear coarsened flag
+        // to eliminate all associated locks/unlocks.
+        this->clear_coarsened();
+      }
       return result;
     }
 
@@ -1546,7 +1561,7 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     // Try lock coarsening
     //
     PhaseIterGVN* iter = phase->is_IterGVN();
-    if (iter != NULL) {
+    if (iter != NULL && !is_eliminated()) {
 
       GrowableArray<AbstractLockNode*>   lock_ops;
 
@@ -1602,7 +1617,7 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
           lock->set_eliminated();
           lock->set_coarsened();
         }
-      } else if (result != NULL && ctrl->is_Region() &&
+      } else if (ctrl->is_Region() &&
                  iter->_worklist.member(ctrl)) {
         // We weren't able to find any opportunities but the region this
         // lock is control dependent on hasn't been processed yet so put
@@ -1623,7 +1638,10 @@ uint UnlockNode::size_of() const { return sizeof(*this); }
 Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
 
   // perform any generic optimizations first (returns 'this' or NULL)
-  Node * result = SafePointNode::Ideal(phase, can_reshape);
+  Node *result = SafePointNode::Ideal(phase, can_reshape);
+  if (result != NULL)  return result;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
 
   // Now see if we can optimize away this unlock.  We don't actually
   // remove the unlocking here, we simply set the _eliminate flag which
@@ -1631,7 +1649,7 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   // modify the graph, the value returned from this function is the
   // one computed above.
   // Escape state is defined after Parse phase.
-  if (result == NULL && can_reshape && EliminateLocks && !is_eliminated()) {
+  if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) {
     //
     // If we are unlocking an unescaped object, the lock/unlock is unnecessary.
     //
@@ -1640,8 +1658,16 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     if (cgr != NULL)
       es = cgr->escape_state(obj_node());
     if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) {
-      // Mark it eliminated to update any counters
+      if (!is_eliminated()) {
-      this->set_eliminated();
+        // Mark it eliminated to update any counters
+        this->set_eliminated();
+      } else {
+        assert(is_coarsened(), "sanity");
+        // The lock could be marked eliminated by lock coarsening
+        // code during first IGVN before EA. Clear coarsened flag
+        // to eliminate all associated locks/unlocks.
+        this->clear_coarsened();
+      }
     }
   }
   return result;

hotspot/src/share/vm/opto/callnode.hpp

@@ -440,6 +440,10 @@ class SafePointScalarObjectNode: public TypeNode {
                      // states of the scalarized object fields are collected.
   uint _n_fields;    // Number of non-static fields of the scalarized object.
   DEBUG_ONLY(AllocateNode* _alloc;)
+
+  virtual uint hash() const ; // { return NO_HASH; }
+  virtual uint cmp( const Node &n ) const;
+
 public:
   SafePointScalarObjectNode(const TypeOopPtr* tp,
 #ifdef ASSERT
@@ -454,15 +458,10 @@ public:
 
   uint first_index() const { return _first_index; }
   uint n_fields()    const { return _n_fields; }
-  DEBUG_ONLY(AllocateNode* alloc() const { return _alloc; })
 
-  // SafePointScalarObject should be always pinned to the control edge
+#ifdef ASSERT
-  // of the SafePoint node for which it was generated.
+  AllocateNode* alloc() const { return _alloc; }
-  virtual bool pinned() const; // { return true; }
+#endif
-
-  // SafePointScalarObject depends on the SafePoint node
-  // for which it was generated.
-  virtual bool depends_only_on_test() const; // { return false; }
 
   virtual uint size_of() const { return sizeof(*this); }
 
@@ -880,6 +879,7 @@ public:
 
   bool is_coarsened()  { return _coarsened; }
   void set_coarsened() { _coarsened = true; }
+  void clear_coarsened() { _coarsened = false; }
 
   // locking does not modify its arguments
   virtual bool        may_modify(const TypePtr *addr_t, PhaseTransform *phase){ return false;}

hotspot/src/share/vm/opto/compile.cpp

@@ -1711,11 +1711,22 @@ void Compile::Optimize() {
 
     if (failing())  return;
 
+    // Optimize out fields loads from scalar replaceable allocations.
     igvn.optimize();
     print_method("Iter GVN after EA", 2);
 
     if (failing())  return;
 
+    if (congraph() != NULL && macro_count() > 0) {
+      PhaseMacroExpand mexp(igvn);
+      mexp.eliminate_macro_nodes();
+      igvn.set_delay_transform(false);
+
+      igvn.optimize();
+      print_method("Iter GVN after eliminating allocations and locks", 2);
+
+      if (failing())  return;
+    }
   }
 
   // Loop transforms on the ideal graph.  Range Check Elimination,

hotspot/src/share/vm/opto/escape.cpp

@@ -1772,12 +1772,20 @@ bool ConnectionGraph::compute_escape() {
       Node *n = C->macro_node(i);
       if (n->is_AbstractLock()) { // Lock and Unlock nodes
         AbstractLockNode* alock = n->as_AbstractLock();
-        if (!alock->is_eliminated()) {
+        if (!alock->is_eliminated() || alock->is_coarsened()) {
           PointsToNode::EscapeState es = escape_state(alock->obj_node());
           assert(es != PointsToNode::UnknownEscape, "should know");
           if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) {
-            // Mark it eliminated
+            if (!alock->is_eliminated()) {
-            alock->set_eliminated();
+              // Mark it eliminated to update any counters
+              alock->set_eliminated();
+            } else {
+              // The lock could be marked eliminated by lock coarsening
+              // code during first IGVN before EA. Clear coarsened flag
+              // to eliminate all associated locks/unlocks and relock
+              // during deoptimization.
+              alock->clear_coarsened();
+            }
           }
         }
       }

hotspot/src/share/vm/opto/gcm.cpp

@@ -95,7 +95,7 @@ void PhaseCFG::replace_block_proj_ctrl( Node *n ) {
   assert(in0 != NULL, "Only control-dependent");
   const Node *p = in0->is_block_proj();
   if (p != NULL && p != n) {    // Control from a block projection?
-    assert(!n->pinned() || n->is_MachConstantBase() || n->is_SafePointScalarObject(), "only pinned MachConstantBase or SafePointScalarObject node is expected here");
+    assert(!n->pinned() || n->is_MachConstantBase(), "only pinned MachConstantBase node is expected here");
     // Find trailing Region
     Block *pb = _bbs[in0->_idx]; // Block-projection already has basic block
     uint j = 0;

hotspot/src/share/vm/opto/loopnode.cpp

@@ -1946,7 +1946,7 @@ void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts)
   }
 
   // Nothing to do, so get out
-  if( !C->has_loops() && !do_split_ifs && !_verify_me && !_verify_only ) {
+  if( !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only ) {
     _igvn.optimize();           // Cleanup NeverBranches
     return;
   }

hotspot/src/share/vm/opto/macro.cpp

@@ -81,7 +81,7 @@ void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcal
       uint old_unique = C->unique();
       Node* new_in = old_sosn->clone(jvms_adj, sosn_map);
       if (old_unique != C->unique()) {
-        new_in->set_req(0, newcall->in(0)); // reset control edge
+        new_in->set_req(0, C->root()); // reset control edge
         new_in = transform_later(new_in); // Register new node.
       }
       old_in = new_in;
@@ -565,7 +565,6 @@ bool PhaseMacroExpand::can_eliminate_allocation(AllocateNode *alloc, GrowableArr
   if (res == NULL) {
     // All users were eliminated.
   } else if (!res->is_CheckCastPP()) {
-    alloc->_is_scalar_replaceable = false;  // don't try again
     NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";)
     can_eliminate = false;
   } else {
@@ -719,7 +718,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
                                                  alloc,
 #endif
                                                  first_ind, nfields);
-    sobj->init_req(0, sfpt->in(TypeFunc::Control));
+    sobj->init_req(0, C->root());
     transform_later(sobj);
 
     // Scan object's fields adding an input to the safepoint for each field.
@@ -762,10 +761,10 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
 
       Node *field_val = value_from_mem(mem, basic_elem_type, field_type, field_addr_type, alloc);
       if (field_val == NULL) {
-        // we weren't able to find a value for this field,
+        // We weren't able to find a value for this field,
-        // give up on eliminating this allocation
+        // give up on eliminating this allocation.
-        alloc->_is_scalar_replaceable = false;  // don't try again
+
-        // remove any extra entries we added to the safepoint
+        // Remove any extra entries we added to the safepoint.
         uint last = sfpt->req() - 1;
         for (int k = 0;  k < j; k++) {
           sfpt->del_req(last--);
@@ -1804,9 +1803,9 @@ bool PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) {
   #ifndef PRODUCT
   if (PrintEliminateLocks) {
     if (alock->is_Lock()) {
-      tty->print_cr("++++ Eliminating: %d Lock", alock->_idx);
+      tty->print_cr("++++ Eliminated: %d Lock", alock->_idx);
     } else {
-      tty->print_cr("++++ Eliminating: %d Unlock", alock->_idx);
+      tty->print_cr("++++ Eliminated: %d Unlock", alock->_idx);
     }
   }
   #endif
@@ -2165,11 +2164,12 @@ void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) {
   _igvn.replace_node(_memproj_fallthrough, mem_phi);
 }
 
-//------------------------------expand_macro_nodes----------------------
+//---------------------------eliminate_macro_nodes----------------------
-//  Returns true if a failure occurred.
+// Eliminate scalar replaced allocations and associated locks.
-bool PhaseMacroExpand::expand_macro_nodes() {
+void PhaseMacroExpand::eliminate_macro_nodes() {
   if (C->macro_count() == 0)
-    return false;
+    return;
+
   // First, attempt to eliminate locks
   int cnt = C->macro_count();
   for (int i=0; i < cnt; i++) {
@@ -2189,14 +2189,6 @@ bool PhaseMacroExpand::expand_macro_nodes() {
       debug_only(int old_macro_count = C->macro_count(););
       if (n->is_AbstractLock()) {
         success = eliminate_locking_node(n->as_AbstractLock());
-      } else if (n->Opcode() == Op_LoopLimit) {
-        // Remove it from macro list and put on IGVN worklist to optimize.
-        C->remove_macro_node(n);
-        _igvn._worklist.push(n);
-        success = true;
-      } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
-        _igvn.replace_node(n, n->in(1));
-        success = true;
       }
       assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
       progress = progress || success;
@@ -2220,18 +2212,50 @@ bool PhaseMacroExpand::expand_macro_nodes() {
         assert(!n->as_AbstractLock()->is_eliminated(), "sanity");
         break;
       default:
-        assert(false, "unknown node type in macro list");
+        assert(n->Opcode() == Op_LoopLimit ||
+               n->Opcode() == Op_Opaque1   ||
+               n->Opcode() == Op_Opaque2, "unknown node type in macro list");
       }
       assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
       progress = progress || success;
     }
   }
+}
+
+//------------------------------expand_macro_nodes----------------------
+//  Returns true if a failure occurred.
+bool PhaseMacroExpand::expand_macro_nodes() {
+  // Last attempt to eliminate macro nodes.
+  eliminate_macro_nodes();
+
   // Make sure expansion will not cause node limit to be exceeded.
   // Worst case is a macro node gets expanded into about 50 nodes.
   // Allow 50% more for optimization.
   if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) )
     return true;
 
+  // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
+  bool progress = true;
+  while (progress) {
+    progress = false;
+    for (int i = C->macro_count(); i > 0; i--) {
+      Node * n = C->macro_node(i-1);
+      bool success = false;
+      debug_only(int old_macro_count = C->macro_count(););
+      if (n->Opcode() == Op_LoopLimit) {
+        // Remove it from macro list and put on IGVN worklist to optimize.
+        C->remove_macro_node(n);
+        _igvn._worklist.push(n);
+        success = true;
+      } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) {
+        _igvn.replace_node(n, n->in(1));
+        success = true;
+      }
+      assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count");
+      progress = progress || success;
+    }
+  }
+
   // expand "macro" nodes
   // nodes are removed from the macro list as they are processed
   while (C->macro_count() > 0) {
@@ -2265,5 +2289,6 @@ bool PhaseMacroExpand::expand_macro_nodes() {
 
   _igvn.set_delay_transform(false);
   _igvn.optimize();
+  if (C->failing())  return true;
   return false;
 }

hotspot/src/share/vm/opto/macro.hpp

@@ -119,6 +119,7 @@ public:
   PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn) {
     _igvn.set_delay_transform(true);
   }
+  void eliminate_macro_nodes();
   bool expand_macro_nodes();
 
 };

hotspot/src/share/vm/opto/memnode.cpp

@@ -2661,6 +2661,8 @@ uint StrIntrinsicNode::match_edge(uint idx) const {
 // control copies
 Node *StrIntrinsicNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (remove_dead_region(phase, can_reshape)) return this;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
 
   if (can_reshape) {
     Node* mem = phase->transform(in(MemNode::Memory));
@@ -2675,6 +2677,12 @@ Node *StrIntrinsicNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   return NULL;
 }
 
+//------------------------------Value------------------------------------------
+const Type *StrIntrinsicNode::Value( PhaseTransform *phase ) const {
+  if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP;
+  return bottom_type();
+}
+
 //=============================================================================
 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
@@ -2715,6 +2723,8 @@ MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
 // control copies
 Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
   if (remove_dead_region(phase, can_reshape)) return this;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
 
   // Eliminate volatile MemBars for scalar replaced objects.
   if (can_reshape && req() == (Precedent+1) &&

hotspot/src/share/vm/opto/memnode.hpp

@@ -800,6 +800,7 @@ public:
   virtual uint match_edge(uint idx) const;
   virtual uint ideal_reg() const { return Op_RegI; }
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
+  virtual const Type *Value(PhaseTransform *phase) const;
 };
 
 //------------------------------StrComp-------------------------------------