> hashtable;
private boolean needsText;
private ElementHandler parentElement;
@@ -110,7 +110,7 @@ public class XMLParser implements ContentHandler {
}
public ElementHandler(String name, boolean needsText) {
- this.hashtable = new Hashtable>();
+ this.hashtable = new HashMap>();
this.name = name;
this.needsText = needsText;
}
@@ -153,7 +153,7 @@ public class XMLParser implements ContentHandler {
for (int i = 0; i < length; i++) {
String val = attr.getValue(i).intern();
String localName = attr.getLocalName(i).intern();
- p.add(new Property(val, localName));
+ p.setProperty(val, localName);
}
}
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Data/src/com/sun/hotspot/igv/data/serialization/XMLWriter.java b/hotspot/src/share/tools/IdealGraphVisualizer/Data/src/com/sun/hotspot/igv/data/serialization/XMLWriter.java
index e112dbf4f07..90876c42482 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Data/src/com/sun/hotspot/igv/data/serialization/XMLWriter.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Data/src/com/sun/hotspot/igv/data/serialization/XMLWriter.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@ public class XMLWriter extends Writer {
inner.write("<" + name);
elementStack.push(name);
- for (Property p : attributes.getProperties()) {
+ for (Property p : attributes) {
inner.write(" " + p.getName() + "=\"");
write(p.getValue().toCharArray());
inner.write("\"");
@@ -101,7 +101,7 @@ public class XMLWriter extends Writer {
public void simpleTag(String name, Properties attributes) throws IOException {
inner.write("<" + name);
- for (Property p : attributes.getProperties()) {
+ for (Property p : attributes) {
inner.write(" " + p.getName() + "=\"");
write(p.getValue().toCharArray());
inner.write("\"");
@@ -111,13 +111,13 @@ public class XMLWriter extends Writer {
}
public void writeProperties(Properties props) throws IOException {
- if (props.getProperties().size() == 0) {
+ if (props.getProperties().hasNext() == false) {
return;
}
startTag(Parser.PROPERTIES_ELEMENT);
- for (Property p : props.getProperties()) {
+ for (Property p : props) {
startTag(Parser.PROPERTY_ELEMENT, new Properties(Parser.PROPERTY_NAME_PROPERTY, p.getName()));
this.write(p.getValue().toCharArray());
endTag();
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Difference/src/com/sun/hotspot/igv/difference/Difference.java b/hotspot/src/share/tools/IdealGraphVisualizer/Difference/src/com/sun/hotspot/igv/difference/Difference.java
index 335344eb77b..776560b15f5 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Difference/src/com/sun/hotspot/igv/difference/Difference.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Difference/src/com/sun/hotspot/igv/difference/Difference.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,6 +29,7 @@ import com.sun.hotspot.igv.data.InputEdge;
import com.sun.hotspot.igv.data.InputGraph;
import com.sun.hotspot.igv.data.InputNode;
import com.sun.hotspot.igv.data.Property;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -124,8 +125,8 @@ public class Difference {
inputNodeMap.put(n, n2);
}
- Set edgesA = a.getEdges();
- Set edgesB = b.getEdges();
+ Collection edgesA = a.getEdges();
+ Collection edgesB = b.getEdges();
Set newEdges = new HashSet();
@@ -182,7 +183,7 @@ public class Difference {
public double getValue() {
double result = 0.0;
- for (Property p : n1.getProperties().getProperties()) {
+ for (Property p : n1.getProperties()) {
double faktor = 1.0;
for (String forbidden : IGNORE_PROPERTIES) {
if (p.getName().equals(forbidden)) {
@@ -287,34 +288,34 @@ public class Difference {
private static void markAsChanged(InputNode n, InputNode firstNode, InputNode otherNode) {
boolean difference = false;
- for (Property p : otherNode.getProperties().getProperties()) {
- String s = firstNode.getProperties().getProperty(p.getName());
+ for (Property p : otherNode.getProperties()) {
+ String s = firstNode.getProperties().get(p.getName());
if (!p.getValue().equals(s)) {
difference = true;
- n.getProperties().add(new Property(OLD_PREFIX + p.getName(), p.getValue()));
+ n.getProperties().setProperty(OLD_PREFIX + p.getName(), p.getValue());
}
}
- for (Property p : firstNode.getProperties().getProperties()) {
- String s = otherNode.getProperties().getProperty(p.getName());
+ for (Property p : firstNode.getProperties()) {
+ String s = otherNode.getProperties().get(p.getName());
if (s == null && p.getValue().length() > 0) {
difference = true;
- n.getProperties().add(new Property(OLD_PREFIX + p.getName(), ""));
+ n.getProperties().setProperty(OLD_PREFIX + p.getName(), "");
}
}
if (difference) {
- n.getProperties().add(new Property(PROPERTY_STATE, VALUE_CHANGED));
+ n.getProperties().setProperty(PROPERTY_STATE, VALUE_CHANGED);
} else {
- n.getProperties().add(new Property(PROPERTY_STATE, VALUE_SAME));
+ n.getProperties().setProperty(PROPERTY_STATE, VALUE_SAME);
}
}
private static void markAsDeleted(InputNode n) {
- n.getProperties().add(new Property(PROPERTY_STATE, VALUE_DELETED));
+ n.getProperties().setProperty(PROPERTY_STATE, VALUE_DELETED);
}
private static void markAsNew(InputNode n) {
- n.getProperties().add(new Property(PROPERTY_STATE, VALUE_NEW));
+ n.getProperties().setProperty(PROPERTY_STATE, VALUE_NEW);
}
}
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/manifest.mf b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/manifest.mf
index 185ff360536..da8d443b487 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/manifest.mf
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/manifest.mf
@@ -1,6 +1,6 @@
-Manifest-Version: 1.0
-OpenIDE-Module: com.sun.hotspot.igv.filter
-OpenIDE-Module-Layer: com/sun/hotspot/igv/filter/layer.xml
-OpenIDE-Module-Localizing-Bundle: com/sun/hotspot/igv/filter/Bundle.properties
-OpenIDE-Module-Specification-Version: 1.0
-
+Manifest-Version: 1.0
+OpenIDE-Module: com.sun.hotspot.igv.filter
+OpenIDE-Module-Layer: com/sun/hotspot/igv/filter/layer.xml
+OpenIDE-Module-Localizing-Bundle: com/sun/hotspot/igv/filter/Bundle.properties
+OpenIDE-Module-Specification-Version: 1.0
+
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/CustomFilter.java b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/CustomFilter.java
index 762d8d2a22e..9704d834a4a 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/CustomFilter.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/CustomFilter.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,6 @@
package com.sun.hotspot.igv.filter;
import com.sun.hotspot.igv.graph.Diagram;
-import com.sun.hotspot.igv.data.Property;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
@@ -56,7 +55,7 @@ public class CustomFilter extends AbstractFilter {
public CustomFilter(String name, String code) {
this.name = name;
this.code = code;
- getProperties().add(new Property("name", name));
+ getProperties().setProperty("name", name);
}
public String getName() {
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/SplitFilter.java b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/SplitFilter.java
index e4c2554bda5..baf1ca8c723 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/SplitFilter.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Filter/src/com/sun/hotspot/igv/filter/SplitFilter.java
@@ -56,8 +56,8 @@ public class SplitFilter extends AbstractFilter {
for (OutputSlot os : f.getOutputSlots()) {
for (Connection c : os.getConnections()) {
InputSlot is = c.getInputSlot();
- is.setName(f.getProperties().getProperty("dump_spec"));
- String s = f.getProperties().getProperty("short_name");
+ is.setName(f.getProperties().get("dump_spec"));
+ String s = f.getProperties().get("short_name");
if (s != null) {
is.setShortName(s);
}
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Diagram.java b/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Diagram.java
index 00d48f6e926..2584b44fd91 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Diagram.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Diagram.java
@@ -35,7 +35,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -126,7 +126,7 @@ public class Diagram {
d.updateBlocks();
Collection nodes = graph.getNodes();
- Hashtable figureHash = new Hashtable();
+ HashMap figureHash = new HashMap();
for (InputNode n : nodes) {
Figure f = d.createFigure();
f.getSource().addSourceNode(n);
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Figure.java b/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Figure.java
index cfb913ca318..d61bb286d3f 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Figure.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Graph/src/com/sun/hotspot/igv/graph/Figure.java
@@ -42,7 +42,7 @@ import java.util.Set;
*
* @author Thomas Wuerthinger
*/
-public class Figure extends Properties.Object implements Source.Provider, Vertex {
+public class Figure extends Properties.Entity implements Source.Provider, Vertex {
public static final int INSET = 6;
public static final int SLOT_WIDTH = 10;
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/Graph.java b/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/Graph.java
index 98dbe392c21..ada9d678617 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/Graph.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/Graph.java
@@ -26,7 +26,7 @@ package com.sun.hotspot.igv.hierarchicallayout;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
@@ -37,13 +37,13 @@ import java.util.Queue;
*/
public class Graph {
- private Hashtable> nodes;
- private Hashtable> edges;
+ private HashMap> nodes;
+ private HashMap> edges;
private List> nodeList;
public Graph() {
- nodes = new Hashtable>();
- edges = new Hashtable>();
+ nodes = new HashMap>();
+ edges = new HashMap>();
nodeList = new ArrayList>();
}
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/HierarchicalClusterLayoutManager.java b/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/HierarchicalClusterLayoutManager.java
index 1dc1b3691d7..37b05d245c1 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/HierarchicalClusterLayoutManager.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/HierarchicalLayout/src/com/sun/hotspot/igv/hierarchicallayout/HierarchicalClusterLayoutManager.java
@@ -25,7 +25,7 @@ package com.sun.hotspot.igv.hierarchicallayout;
import java.awt.Point;
import java.awt.Rectangle;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.ArrayList;
@@ -69,19 +69,19 @@ public class HierarchicalClusterLayoutManager implements LayoutManager {
assert graph.verify();
- Hashtable> lists = new Hashtable>();
- Hashtable> listsConnection = new Hashtable>();
- Hashtable> clusterInputSlotHash = new Hashtable>();
- Hashtable> clusterOutputSlotHash = new Hashtable>();
+ HashMap> lists = new HashMap>();
+ HashMap> listsConnection = new HashMap>();
+ HashMap> clusterInputSlotHash = new HashMap>();
+ HashMap> clusterOutputSlotHash = new HashMap>();
- Hashtable clusterNodes = new Hashtable();
- Hashtable> clusterInputSlotSet = new Hashtable>();
- Hashtable> clusterOutputSlotSet = new Hashtable>();
+ HashMap clusterNodes = new HashMap();
+ HashMap> clusterInputSlotSet = new HashMap>();
+ HashMap> clusterOutputSlotSet = new HashMap>();
Set clusterEdges = new HashSet ();
Set interClusterEdges = new HashSet ();
- Hashtable linkClusterOutgoingConnection = new Hashtable ();
- Hashtable linkInterClusterConnection = new Hashtable ();
- Hashtable linkClusterIngoingConnection = new Hashtable ();
+ HashMap linkClusterOutgoingConnection = new HashMap ();
+ HashMap linkInterClusterConnection = new HashMap ();
+ HashMap linkClusterIngoingConnection = new HashMap ();
Set clusterNodeSet = new HashSet();
Set cluster = graph.getClusters();
@@ -89,8 +89,8 @@ public class HierarchicalClusterLayoutManager implements LayoutManager {
for (Cluster c : cluster) {
lists.put(c, new ArrayList());
listsConnection.put(c, new ArrayList ());
- clusterInputSlotHash.put(c, new Hashtable());
- clusterOutputSlotHash.put(c, new Hashtable());
+ clusterInputSlotHash.put(c, new HashMap());
+ clusterOutputSlotHash.put(c, new HashMap());
clusterOutputSlotSet.put(c, new TreeSet());
clusterInputSlotSet.put(c, new TreeSet());
ClusterNode cn = new ClusterNode(c, "" + z);
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Layout/src/com/sun/hotspot/igv/layout/LayoutGraph.java b/hotspot/src/share/tools/IdealGraphVisualizer/Layout/src/com/sun/hotspot/igv/layout/LayoutGraph.java
index ac262c95e74..50edc14a864 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Layout/src/com/sun/hotspot/igv/layout/LayoutGraph.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Layout/src/com/sun/hotspot/igv/layout/LayoutGraph.java
@@ -24,7 +24,7 @@
package com.sun.hotspot.igv.layout;
import java.util.HashSet;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -37,9 +37,9 @@ public class LayoutGraph {
private Set extends Link> links;
private SortedSet vertices;
- private Hashtable> inputPorts;
- private Hashtable> outputPorts;
- private Hashtable> portLinks;
+ private HashMap> inputPorts;
+ private HashMap> outputPorts;
+ private HashMap> portLinks;
public LayoutGraph(Set extends Link> links) {
this(links, new HashSet());
@@ -50,9 +50,9 @@ public class LayoutGraph {
assert verify();
vertices = new TreeSet();
- portLinks = new Hashtable>();
- inputPorts = new Hashtable>();
- outputPorts = new Hashtable>();
+ portLinks = new HashMap>();
+ inputPorts = new HashMap>();
+ outputPorts = new HashMap>();
for (Link l : links) {
Port p = l.getFrom();
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/README b/hotspot/src/share/tools/IdealGraphVisualizer/README
index 80e16329506..5c9a16601bc 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/README
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/README
@@ -5,21 +5,16 @@ of Linz in Austria and has been included as part of hotspot since that
was the primary target of the tool. The tool itself is fairly general
with only a few modules that contain C2 specific elements.
-The tool is built on top of the NetBeans 6.0 rich client
+The tool is built on top of the NetBeans 6.1 rich client
infrastructure and so requires NetBeans to build. It currently
requires Java 6 to run as it needs support for JavaScript for its
filtering mechanism and assumes it's built into the platform. It
-should build out of the box whit NetBeans 6 and Java 6 or later. It's
-possible to run it on 1.5 by including Rhino on the classpath though
-that currently isn't working correctly. Support for exporting graphs
-as SVG can be enabled by adding batik to the classpath which isn't
-included by default.
-
-It can be built on top of NetBeans 6.1 if you change the required
-modules to be platform8 instead of platform7. The tool could run on
-JDK 1.5 with some reworking of the how the JavaScript support is
-enabled but currently it requires some tweaking of the setup. This
-will be fixed in a later setup.
+should build out of the box with NetBeans 6.1 and Java 6 or later.
+It's possible to run it on 1.5 by including Rhino on the classpath
+though that currently isn't working correctly. Support for exporting
+graphs as SVG can be enabled by adding batik to the classpath which
+isn't included by default. It can be built on top of NetBeans 6.0 if
+you change the required modules to be platform7 instead of platform8.
The JVM support is controlled by the flag -XX:PrintIdealGraphLevel=#
where # is:
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/PropertiesSheet.java b/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/PropertiesSheet.java
index f058edda6ae..6eca3e9517b 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/PropertiesSheet.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/PropertiesSheet.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,11 +36,11 @@ import org.openide.nodes.Sheet;
*/
public class PropertiesSheet {
- public static void initializeSheet(Properties properties, Sheet s) {
+ public static void initializeSheet(final Properties properties, Sheet s) {
Sheet.Set set1 = Sheet.createPropertiesSet();
set1.setDisplayName("Properties");
- for (final Property p : properties.getProperties()) {
+ for (final Property p : properties) {
Node.Property prop = new Node.Property(String.class) {
@Override
@@ -60,7 +60,7 @@ public class PropertiesSheet {
@Override
public void setValue(String arg0) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException {
- p.setValue(arg0);
+ properties.setProperty(p.getName(), arg0);
}
};
prop.setName(p.getName());
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/RangeSliderModel.java b/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/RangeSliderModel.java
index 71766ec3b67..5c1d9ee31fe 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/RangeSliderModel.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/Util/src/com/sun/hotspot/igv/util/RangeSliderModel.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -65,13 +65,19 @@ public class RangeSliderModel implements ChangedEventProvider
public RangeSliderModel(List positions) {
assert positions.size() > 0;
- this.positions = positions;
this.changedEvent = new ChangedEvent(this);
this.colorChangedEvent = new ChangedEvent(this);
+ setPositions(positions);
+ }
+
+ protected void setPositions(List positions) {
+ this.positions = positions;
colors = new ArrayList();
for (int i = 0; i < positions.size(); i++) {
colors.add(Color.black);
}
+ changedEvent.fire();
+ colorChangedEvent.fire();
}
public void setColors(List colors) {
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramScene.java b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramScene.java
index 2ee3067acfe..7ebfbbe866a 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramScene.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramScene.java
@@ -63,7 +63,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Hashtable;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -104,10 +104,10 @@ import org.openide.util.lookup.InstanceContent;
*/
public class DiagramScene extends Scene implements ChangedListener {
- private Hashtable figureWidgets;
- private Hashtable slotWidgets;
- private Hashtable connectionWidgets;
- private Hashtable blockWidgets;
+ private HashMap figureWidgets;
+ private HashMap slotWidgets;
+ private HashMap connectionWidgets;
+ private HashMap blockWidgets;
private Widget hoverWidget;
private WidgetAction hoverAction;
private List selectedWidgets;
@@ -414,7 +414,7 @@ public class DiagramScene extends Scene implements ChangedListener();
+ blockWidgets = new HashMap();
boolean b = this.getUndoRedoEnabled();
this.setUndoRedoEnabled(false);
@@ -543,9 +543,9 @@ public class DiagramScene extends Scene implements ChangedListener();
- slotWidgets = new Hashtable();
- connectionWidgets = new Hashtable();
+ figureWidgets = new HashMap();
+ slotWidgets = new HashMap();
+ connectionWidgets = new HashMap();
WidgetAction selectAction = new ExtendedSelectAction(selectProvider);
Diagram d = getModel().getDiagramToView();
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramViewModel.java b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramViewModel.java
index e374873b584..0db439650e2 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramViewModel.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/DiagramViewModel.java
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -55,6 +55,7 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
private FilterChain filterChain;
private FilterChain sequenceFilterChain;
private Diagram diagram;
+ private ChangedEvent groupChangedEvent;
private ChangedEvent diagramChangedEvent;
private ChangedEvent viewChangedEvent;
private ChangedEvent viewPropertiesChangedEvent;
@@ -67,6 +68,7 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
}
};
+ @Override
public DiagramViewModel copy() {
DiagramViewModel result = new DiagramViewModel(group, filterChain, sequenceFilterChain);
result.setData(this);
@@ -79,6 +81,7 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
boolean viewChanged = false;
boolean viewPropertiesChanged = false;
+ boolean groupChanged = (group == newModel.group);
this.group = newModel.group;
diagramChanged |= (filterChain != newModel.filterChain);
this.filterChain = newModel.filterChain;
@@ -97,6 +100,10 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
viewPropertiesChanged |= (showNodeHull != newModel.showNodeHull);
this.showNodeHull = newModel.showNodeHull;
+ if(groupChanged) {
+ groupChangedEvent.fire();
+ }
+
if (diagramChanged) {
diagramChangedEvent.fire();
}
@@ -143,11 +150,38 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
diagramChangedEvent = new ChangedEvent(this);
viewChangedEvent = new ChangedEvent(this);
viewPropertiesChangedEvent = new ChangedEvent(this);
+ groupChangedEvent = new ChangedEvent(this);
+ groupChangedEvent.addListener(groupChangedListener);
+ groupChangedEvent.fire();
filterChain.getChangedEvent().addListener(filterChainChangedListener);
sequenceFilterChain.getChangedEvent().addListener(filterChainChangedListener);
}
+ private final ChangedListener groupChangedListener = new ChangedListener() {
+
+ private Group oldGroup;
+
+ public void changed(DiagramViewModel source) {
+ if(oldGroup != null) {
+ oldGroup.getChangedEvent().removeListener(groupContentChangedListener);
+ }
+ group.getChangedEvent().addListener(groupContentChangedListener);
+ oldGroup = group;
+ }
+ };
+
+
+ private final ChangedListener groupContentChangedListener = new ChangedListener() {
+
+ public void changed(Group source) {
+ assert source == group;
+ setPositions(calculateStringList(source));
+ setSelectedNodes(selectedNodes);
+ }
+
+ };
+
public ChangedEvent getDiagramChangedEvent() {
return diagramChangedEvent;
}
@@ -268,7 +302,10 @@ public class DiagramViewModel extends RangeSliderModel implements ChangedListene
}
public InputGraph getSecondGraph() {
- return group.getGraphs().get(getSecondPosition());
+ List graphs = group.getGraphs();
+ if (graphs.size() >= getSecondPosition())
+ return group.getGraphs().get(getSecondPosition());
+ return getFirstGraph();
}
public void selectGraph(InputGraph g) {
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/FindPanel.java b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/FindPanel.java
index 7d45f1b35c6..961dc54d882 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/FindPanel.java
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/View/src/com/sun/hotspot/igv/view/FindPanel.java
@@ -67,7 +67,7 @@ class FindPanel extends JPanel implements KeyListener {
for (Figure f : figures) {
Properties prop = f.getProperties();
- for (Property p : prop.getProperties()) {
+ for (Property p : prop) {
if (!propertyNames.contains(p.getName())) {
propertyNames.add(p.getName());
}
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/platform.properties b/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/platform.properties
index 4768730f129..881365d0eda 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/platform.properties
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/platform.properties
@@ -1,16 +1,16 @@
# Deprecated since 5.0u1; for compatibility with 5.0:
disabled.clusters=\
apisupport1,\
+ gsf1,\
harness,\
- ide8,\
- java1,\
- nb6.0,\
- profiler2
+ ide9,\
+ java2,\
+ nb6.1,\
+ profiler3
disabled.modules=\
org.netbeans.core.execution,\
org.netbeans.core.multiview,\
org.netbeans.core.output2,\
- org.netbeans.modules.applemenu,\
org.netbeans.modules.autoupdate.services,\
org.netbeans.modules.autoupdate.ui,\
org.netbeans.modules.core.kit,\
@@ -24,6 +24,6 @@ disabled.modules=\
org.openide.execution,\
org.openide.util.enumerations
enabled.clusters=\
- platform7
+ platform8
nbjdk.active=default
nbplatform.active=default
diff --git a/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/project.properties b/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/project.properties
index 1b524d231f3..b00f22d78fc 100644
--- a/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/project.properties
+++ b/hotspot/src/share/tools/IdealGraphVisualizer/nbproject/project.properties
@@ -15,7 +15,6 @@ modules=\
${project.com.sun.hotspot.igv.difference}:\
${project.com.sun.hotspot.igv.settings}:\
${project.com.sun.hotspot.igv.util}:\
- ${project.com.sun.hotspot.igv.rhino}:\
${project.com.sun.hotspot.igv.svg}:\
${project.com.sun.hotspot.connection}:\
${project.com.sun.hotspot.igv.servercompilerscheduler}:\
@@ -31,10 +30,10 @@ project.com.sun.hotspot.igv.filterwindow=FilterWindow
project.com.sun.hotspot.igv.graph=Graph
project.com.sun.hotspot.igv.hierarchicallayout=HierarchicalLayout
project.com.sun.hotspot.igv.layout=Layout
-project.com.sun.hotspot.igv.rhino=RhinoScriptEngineProxy
project.com.sun.hotspot.igv.servercompilerscheduler=ServerCompiler
project.com.sun.hotspot.igv.settings=Settings
project.com.sun.hotspot.igv.svg=BatikSVGProxy
project.com.sun.hotspot.igv.view=View
project.com.sun.hotspot.igv.util=Util
-run.args = -server -J-Xms64m -J-Xmx512m -J-da
+run.args = -J-server -J-Xms64m -J-Xmx1g -J-da
+run.args.extra = -J-server -J-Xms64m -J-Xmx1g -J-da
diff --git a/hotspot/src/share/vm/adlc/adlparse.cpp b/hotspot/src/share/vm/adlc/adlparse.cpp
index 6ebef89f27f..81a95e861ab 100644
--- a/hotspot/src/share/vm/adlc/adlparse.cpp
+++ b/hotspot/src/share/vm/adlc/adlparse.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1997-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,6 @@ ADLParser::ADLParser(FileBuff& buffer, ArchDesc& archDesc)
_globalNames(archDesc.globalNames()) {
_AD._syntax_errs = _AD._semantic_errs = 0; // No errors so far this file
_AD._warnings = 0; // No warnings either
- _linenum = 0; // Will increment to first line
_curline = _ptr = NULL; // No pointers into buffer yet
_preproc_depth = 0;
@@ -76,7 +75,7 @@ ADLParser::~ADLParser() {
}
if (!_AD._quiet_mode)
fprintf(stderr,"-----------------------------------------------------------------------------\n");
- _AD._TotalLines += _linenum-1; // -1 for overshoot in "nextline" routine
+ _AD._TotalLines += linenum()-1; // -1 for overshoot in "nextline" routine
// Write out information we have stored
// // UNIXism == fsync(stderr);
@@ -109,6 +108,7 @@ void ADLParser::parse() {
else if (!strcmp(ident, "pipeline")) pipe_parse();
else if (!strcmp(ident, "definitions")) definitions_parse();
else if (!strcmp(ident, "peephole")) peep_parse();
+ else if (!strcmp(ident, "#line")) preproc_line();
else if (!strcmp(ident, "#define")) preproc_define();
else if (!strcmp(ident, "#undef")) preproc_undef();
else {
@@ -148,7 +148,7 @@ void ADLParser::instr_parse(void) {
if( (ident = get_unique_ident(_globalNames,"instruction")) == NULL )
return;
instr = new InstructForm(ident); // Create new instruction form
- instr->_linenum = _linenum;
+ instr->_linenum = linenum();
_globalNames.Insert(ident, instr); // Add name to the name table
// Debugging Stuff
if (_AD._adl_debug > 1)
@@ -404,7 +404,7 @@ void ADLParser::oper_parse(void) {
if( (ident = get_unique_ident(_globalNames,"operand")) == NULL )
return;
oper = new OperandForm(ident); // Create new operand form
- oper->_linenum = _linenum;
+ oper->_linenum = linenum();
_globalNames.Insert(ident, oper); // Add name to the name table
// Debugging Stuff
@@ -774,7 +774,7 @@ void ADLParser::reg_parse(void) {
// Create the RegisterForm for the architecture description.
RegisterForm *regBlock = new RegisterForm(); // Build new Source object
- regBlock->_linenum = _linenum;
+ regBlock->_linenum = linenum();
_AD.addForm(regBlock);
skipws(); // Skip leading whitespace
@@ -787,9 +787,11 @@ void ADLParser::reg_parse(void) {
parse_err(SYNERR, "missing identifier inside register block.\n");
return;
}
- if (strcmp(token,"reg_def")==0) { reg_def_parse(); }
- if (strcmp(token,"reg_class")==0) { reg_class_parse(); }
- if (strcmp(token,"alloc_class")==0) { alloc_class_parse(); }
+ if (strcmp(token,"reg_def")==0) { reg_def_parse(); }
+ else if (strcmp(token,"reg_class")==0) { reg_class_parse(); }
+ else if (strcmp(token,"alloc_class")==0) { alloc_class_parse(); }
+ else if (strcmp(token,"#define")==0) { preproc_define(); }
+ else { parse_err(SYNERR, "bad token %s inside register block.\n", token); break; }
skipws();
}
}
@@ -847,7 +849,7 @@ void ADLParser::enc_class_parse(void) {
}
EncClass *encoding = _AD._encode->add_EncClass(ec_name);
- encoding->_linenum = _linenum;
+ encoding->_linenum = linenum();
skipws(); // Skip leading whitespace
// Check for optional parameter list
@@ -904,11 +906,7 @@ void ADLParser::enc_class_parse_block(EncClass* encoding, char* ec_name) {
skipws_no_preproc(); // Skip leading whitespace
// Prepend location descriptor, for debugging; cf. ADLParser::find_cpp_block
if (_AD._adlocation_debug) {
- const char* file = _AD._ADL_file._name;
- int line = _linenum;
- char* location = (char *)malloc(strlen(file) + 100);
- sprintf(location, "#line %d \"%s\"\n", line, file);
- encoding->add_code(location);
+ encoding->add_code(get_line_string());
}
// Collect the parts of the encode description
@@ -949,6 +947,10 @@ void ADLParser::enc_class_parse_block(EncClass* encoding, char* ec_name) {
skipws();
+ if (_AD._adlocation_debug) {
+ encoding->add_code(end_line_marker());
+ }
+
// Debug Stuff
if (_AD._adl_debug > 1) fprintf(stderr,"EncodingClass Form: %s\n", ec_name);
}
@@ -2350,7 +2352,11 @@ void ADLParser::reg_class_parse(void) {
return;
}
RegDef *regDef = _AD._register->getRegDef(rname);
- reg_class->addReg(regDef); // add regDef to regClass
+ if (!regDef) {
+ parse_err(SEMERR, "unknown identifier %s inside reg_class list.\n", rname);
+ } else {
+ reg_class->addReg(regDef); // add regDef to regClass
+ }
// Check for ',' and position to next token.
skipws();
@@ -2747,7 +2753,8 @@ Predicate *ADLParser::pred_parse(void) {
char *rule = NULL; // String representation of predicate
skipws(); // Skip leading whitespace
- if ( (rule = get_paren_expr("pred expression")) == NULL ) {
+ int line = linenum();
+ if ( (rule = get_paren_expr("pred expression", true)) == NULL ) {
parse_err(SYNERR, "incorrect or missing expression for 'predicate'\n");
return NULL;
}
@@ -2776,7 +2783,7 @@ InsEncode *ADLParser::ins_encode_parse_block(InstructForm &inst) {
assert(_AD._encode->encClass(ec_name) == NULL, "shouldn't already exist");
EncClass *encoding = _AD._encode->add_EncClass(ec_name);
- encoding->_linenum = _linenum;
+ encoding->_linenum = linenum();
// synthesize the arguments list for the enc_class from the
// arguments to the instruct definition.
@@ -2852,7 +2859,7 @@ InsEncode *ADLParser::ins_encode_parse(InstructForm &inst) {
skipws();
InsEncode *encrule = new InsEncode(); // Encode class for instruction
- encrule->_linenum = _linenum;
+ encrule->_linenum = linenum();
char *ec_name = NULL; // String representation of encode rule
// identifier is optional.
while (_curchar != ')') {
@@ -3203,6 +3210,12 @@ Interface *ADLParser::cond_interface_parse(void) {
char *greater_equal;
char *less_equal;
char *greater;
+ const char *equal_format = "eq";
+ const char *not_equal_format = "ne";
+ const char *less_format = "lt";
+ const char *greater_equal_format = "ge";
+ const char *less_equal_format = "le";
+ const char *greater_format = "gt";
if (_curchar != '%') {
parse_err(SYNERR, "Missing '%{' for 'cond_interface' block.\n");
@@ -3222,22 +3235,22 @@ Interface *ADLParser::cond_interface_parse(void) {
return NULL;
}
if ( strcmp(field,"equal") == 0 ) {
- equal = interface_field_parse();
+ equal = interface_field_parse(&equal_format);
}
else if ( strcmp(field,"not_equal") == 0 ) {
- not_equal = interface_field_parse();
+ not_equal = interface_field_parse(¬_equal_format);
}
else if ( strcmp(field,"less") == 0 ) {
- less = interface_field_parse();
+ less = interface_field_parse(&less_format);
}
else if ( strcmp(field,"greater_equal") == 0 ) {
- greater_equal = interface_field_parse();
+ greater_equal = interface_field_parse(&greater_equal_format);
}
else if ( strcmp(field,"less_equal") == 0 ) {
- less_equal = interface_field_parse();
+ less_equal = interface_field_parse(&less_equal_format);
}
else if ( strcmp(field,"greater") == 0 ) {
- greater = interface_field_parse();
+ greater = interface_field_parse(&greater_format);
}
else {
parse_err(SYNERR, "Expected keyword, base|index|scale|disp, or '%}' ending interface.\n");
@@ -3252,14 +3265,18 @@ Interface *ADLParser::cond_interface_parse(void) {
next_char(); // Skip '}'
// Construct desired object and return
- Interface *inter = new CondInterface(equal, not_equal, less, greater_equal,
- less_equal, greater);
+ Interface *inter = new CondInterface(equal, equal_format,
+ not_equal, not_equal_format,
+ less, less_format,
+ greater_equal, greater_equal_format,
+ less_equal, less_equal_format,
+ greater, greater_format);
return inter;
}
//------------------------------interface_field_parse--------------------------
-char *ADLParser::interface_field_parse(void) {
+char *ADLParser::interface_field_parse(const char ** format) {
char *iface_field = NULL;
// Get interface field
@@ -3280,6 +3297,32 @@ char *ADLParser::interface_field_parse(void) {
return NULL;
}
skipws();
+ if (format != NULL && _curchar == ',') {
+ next_char();
+ skipws();
+ if (_curchar != '"') {
+ parse_err(SYNERR, "Missing '\"' in field format .\n");
+ return NULL;
+ }
+ next_char();
+ char *start = _ptr; // Record start of the next string
+ while ((_curchar != '"') && (_curchar != '%') && (_curchar != '\n')) {
+ if (_curchar == '\\') next_char(); // superquote
+ if (_curchar == '\n') parse_err(SYNERR, "newline in string"); // unimplemented!
+ next_char();
+ }
+ if (_curchar != '"') {
+ parse_err(SYNERR, "Missing '\"' at end of field format .\n");
+ return NULL;
+ }
+ // If a string was found, terminate it and record in FormatRule
+ if ( start != _ptr ) {
+ *_ptr = '\0'; // Terminate the string
+ *format = start;
+ }
+ next_char();
+ skipws();
+ }
if (_curchar != ')') {
parse_err(SYNERR, "Missing ')' after interface field.\n");
return NULL;
@@ -3342,6 +3385,12 @@ FormatRule* ADLParser::format_parse(void) {
next_char(); // Move past the '{'
skipws();
+ if (_curchar == '$') {
+ char* ident = get_rep_var_ident();
+ if (strcmp(ident, "$$template") == 0) return template_parse();
+ parse_err(SYNERR, "Unknown \"%s\" directive in format", ident);
+ return NULL;
+ }
// Check for the opening '"' inside the format description
if ( _curchar == '"' ) {
next_char(); // Move past the initial '"'
@@ -3366,7 +3415,12 @@ FormatRule* ADLParser::format_parse(void) {
// Check if there is a string to pass through to output
char *start = _ptr; // Record start of the next string
while ((_curchar != '$') && (_curchar != '"') && (_curchar != '%') && (_curchar != '\n')) {
- if (_curchar == '\\') next_char(); // superquote
+ if (_curchar == '\\') {
+ next_char(); // superquote
+ if ((_curchar == '$') || (_curchar == '%'))
+ // hack to avoid % escapes and warnings about undefined \ escapes
+ *(_ptr-1) = _curchar;
+ }
if (_curchar == '\n') parse_err(SYNERR, "newline in string"); // unimplemented!
next_char();
}
@@ -3433,6 +3487,131 @@ FormatRule* ADLParser::format_parse(void) {
}
+//------------------------------template_parse-----------------------------------
+FormatRule* ADLParser::template_parse(void) {
+ char *desc = NULL;
+ FormatRule *format = (new FormatRule(desc));
+
+ skipws();
+ while ( (_curchar != '%') && (*(_ptr+1) != '}') ) {
+
+ // (1)
+ // Check if there is a string to pass through to output
+ char *start = _ptr; // Record start of the next string
+ while ((_curchar != '$') && ((_curchar != '%') || (*(_ptr+1) != '}')) ) {
+ // If at the start of a comment, skip past it
+ if( (_curchar == '/') && ((*(_ptr+1) == '/') || (*(_ptr+1) == '*')) ) {
+ skipws_no_preproc();
+ } else {
+ // ELSE advance to the next character, or start of the next line
+ next_char_or_line();
+ }
+ }
+ // If a string was found, terminate it and record in EncClass
+ if ( start != _ptr ) {
+ *_ptr = '\0'; // Terminate the string
+ // Add flag to _strings list indicating we should check _rep_vars
+ format->_strings.addName(NameList::_signal2);
+ format->_strings.addName(start);
+ }
+
+ // (2)
+ // If we are at a replacement variable,
+ // copy it and record in EncClass
+ if ( _curchar == '$' ) {
+ // Found replacement Variable
+ char *rep_var = get_rep_var_ident_dup();
+ if (strcmp(rep_var, "$emit") == 0) {
+ // switch to normal format parsing
+ next_char();
+ next_char();
+ skipws();
+ // Check for the opening '"' inside the format description
+ if ( _curchar == '"' ) {
+ next_char(); // Move past the initial '"'
+ if( _curchar == '"' ) { // Handle empty format string case
+ *_ptr = '\0'; // Terminate empty string
+ format->_strings.addName(_ptr);
+ }
+
+ // Collect the parts of the format description
+ // (1) strings that are passed through to tty->print
+ // (2) replacement/substitution variable, preceeded by a '$'
+ // (3) multi-token ANSIY C style strings
+ while ( true ) {
+ if ( _curchar == '%' || _curchar == '\n' ) {
+ parse_err(SYNERR, "missing '\"' at end of format block");
+ return NULL;
+ }
+
+ // (1)
+ // Check if there is a string to pass through to output
+ char *start = _ptr; // Record start of the next string
+ while ((_curchar != '$') && (_curchar != '"') && (_curchar != '%') && (_curchar != '\n')) {
+ if (_curchar == '\\') next_char(); // superquote
+ if (_curchar == '\n') parse_err(SYNERR, "newline in string"); // unimplemented!
+ next_char();
+ }
+ // If a string was found, terminate it and record in FormatRule
+ if ( start != _ptr ) {
+ *_ptr = '\0'; // Terminate the string
+ format->_strings.addName(start);
+ }
+
+ // (2)
+ // If we are at a replacement variable,
+ // copy it and record in FormatRule
+ if ( _curchar == '$' ) {
+ next_char(); // Move past the '$'
+ char* rep_var = get_ident(); // Nil terminate the variable name
+ rep_var = strdup(rep_var);// Copy the string
+ *_ptr = _curchar; // and replace Nil with original character
+ format->_rep_vars.addName(rep_var);
+ // Add flag to _strings list indicating we should check _rep_vars
+ format->_strings.addName(NameList::_signal);
+ }
+
+ // (3)
+ // Allow very long strings to be broken up,
+ // using the ANSI C syntax "foo\n" "bar"
+ if ( _curchar == '"') {
+ next_char(); // Move past the '"'
+ skipws(); // Skip white space before next string token
+ if ( _curchar != '"') {
+ break;
+ } else {
+ // Found one. Skip both " and the whitespace in between.
+ next_char();
+ }
+ }
+ } // end while part of format description
+ }
+ } else {
+ // Add flag to _strings list indicating we should check _rep_vars
+ format->_rep_vars.addName(rep_var);
+ // Add flag to _strings list indicating we should check _rep_vars
+ format->_strings.addName(NameList::_signal3);
+ }
+ } // end while part of format description
+ }
+
+ skipws();
+ // Past format description, at '%'
+ if ( _curchar != '%' || *(_ptr+1) != '}' ) {
+ parse_err(SYNERR, "missing '%}' at end of format block");
+ return NULL;
+ }
+ next_char(); // Move past the '%'
+ next_char(); // Move past the '}'
+
+ // Debug Stuff
+ if (_AD._adl_debug > 1) fprintf(stderr,"Format Rule: %s\n", desc);
+
+ skipws();
+ return format;
+}
+
+
//------------------------------effect_parse-----------------------------------
void ADLParser::effect_parse(InstructForm *instr) {
char* desc = NULL;
@@ -3776,8 +3955,7 @@ char* ADLParser::find_cpp_block(const char* description) {
next_char(); // Skip block delimiter
skipws_no_preproc(); // Skip leading whitespace
cppBlock = _ptr; // Point to start of expression
- const char* file = _AD._ADL_file._name;
- int line = _linenum;
+ int line = linenum();
next = _ptr + 1;
while(((_curchar != '%') || (*next != '}')) && (_curchar != '\0')) {
next_char_or_line();
@@ -3792,15 +3970,16 @@ char* ADLParser::find_cpp_block(const char* description) {
_curchar = *_ptr; // Maintain invariant
// Prepend location descriptor, for debugging.
- char* location = (char *)malloc(strlen(file) + 100);
- *location = '\0';
- if (_AD._adlocation_debug)
- sprintf(location, "#line %d \"%s\"\n", line, file);
- char* result = (char *)malloc(strlen(location) + strlen(cppBlock) + 1);
- strcpy(result, location);
- strcat(result, cppBlock);
- cppBlock = result;
- free(location);
+ if (_AD._adlocation_debug) {
+ char* location = get_line_string(line);
+ char* end_loc = end_line_marker();
+ char* result = (char *)malloc(strlen(location) + strlen(cppBlock) + strlen(end_loc) + 1);
+ strcpy(result, location);
+ strcat(result, cppBlock);
+ strcat(result, end_loc);
+ cppBlock = result;
+ free(location);
+ }
}
return cppBlock;
@@ -3870,13 +4049,26 @@ char* ADLParser::get_expr(const char *desc, const char *stop_chars) {
// Helper function around get_expr
// Sets _curchar to '(' so that get_paren_expr will search for a matching ')'
-char *ADLParser::get_paren_expr(const char *description) {
+char *ADLParser::get_paren_expr(const char *description, bool include_location) {
+ int line = linenum();
if (_curchar != '(') // Escape if not valid starting position
return NULL;
next_char(); // Skip the required initial paren.
char *token2 = get_expr(description, ")");
if (_curchar == ')')
next_char(); // Skip required final paren.
+ int junk = 0;
+ if (include_location && _AD._adlocation_debug && !is_int_token(token2, junk)) {
+ // Prepend location descriptor, for debugging.
+ char* location = get_line_string(line);
+ char* end_loc = end_line_marker();
+ char* result = (char *)malloc(strlen(location) + strlen(token2) + strlen(end_loc) + 1);
+ strcpy(result, location);
+ strcat(result, token2);
+ strcat(result, end_loc);
+ token2 = result;
+ free(location);
+ }
return token2;
}
@@ -3916,10 +4108,16 @@ char *ADLParser::get_ident_common(bool do_preproc) {
if (do_preproc && start != NULL) {
const char* def = _AD.get_preproc_def(start);
if (def != NULL && strcmp(def, start)) {
- const char* def2 = _AD.get_preproc_def(def);
- if (def2 != NULL && strcmp(def2, def)) {
- parse_err(SYNERR, "unimplemented: using %s defined as %s => %s",
- start, def, def2);
+ const char* def1 = def;
+ const char* def2 = _AD.get_preproc_def(def1);
+ // implement up to 2 levels of #define
+ if (def2 != NULL && strcmp(def2, def1)) {
+ def = def2;
+ const char* def3 = _AD.get_preproc_def(def2);
+ if (def3 != NULL && strcmp(def3, def2) && strcmp(def3, def1)) {
+ parse_err(SYNERR, "unimplemented: using %s defined as %s => %s => %s",
+ start, def1, def2, def3);
+ }
}
start = strdup(def);
}
@@ -4265,6 +4463,35 @@ void ADLParser::get_effectlist(FormDict &effects, FormDict &operands) {
}
+//-------------------------------preproc_line----------------------------------
+// A "#line" keyword has been seen, so parse the rest of the line.
+void ADLParser::preproc_line(void) {
+ int line = get_int();
+ skipws_no_preproc();
+ const char* file = NULL;
+ if (_curchar == '"') {
+ next_char(); // Move past the initial '"'
+ file = _ptr;
+ while (true) {
+ if (_curchar == '\n') {
+ parse_err(SYNERR, "missing '\"' at end of #line directive");
+ return;
+ }
+ if (_curchar == '"') {
+ *_ptr = '\0'; // Terminate the string
+ next_char();
+ skipws_no_preproc();
+ break;
+ }
+ next_char();
+ }
+ }
+ ensure_end_of_line();
+ if (file != NULL)
+ _AD._ADL_file._name = file;
+ _buf.set_linenum(line);
+}
+
//------------------------------preproc_define---------------------------------
// A "#define" keyword has been seen, so parse the rest of the line.
void ADLParser::preproc_define(void) {
@@ -4297,11 +4524,11 @@ void ADLParser::parse_err(int flag, const char *fmt, ...) {
va_start(args, fmt);
if (flag == 1)
- _AD._syntax_errs += _AD.emit_msg(0, flag, _linenum, fmt, args);
+ _AD._syntax_errs += _AD.emit_msg(0, flag, linenum(), fmt, args);
else if (flag == 2)
- _AD._semantic_errs += _AD.emit_msg(0, flag, _linenum, fmt, args);
+ _AD._semantic_errs += _AD.emit_msg(0, flag, linenum(), fmt, args);
else
- _AD._warnings += _AD.emit_msg(0, flag, _linenum, fmt, args);
+ _AD._warnings += _AD.emit_msg(0, flag, linenum(), fmt, args);
int error_char = _curchar;
char* error_ptr = _ptr+1;
@@ -4328,6 +4555,7 @@ void ADLParser::parse_err(int flag, const char *fmt, ...) {
// A preprocessor directive has been encountered. Be sure it has fallen at
// the begining of a line, or else report an error.
void ADLParser::ensure_start_of_line(void) {
+ if (_curchar == '\n') { next_line(); return; }
assert( _ptr >= _curline && _ptr < _curline+strlen(_curline),
"Must be able to find which line we are in" );
@@ -4496,6 +4724,7 @@ char ADLParser::cur_char() {
//---------------------------next_char-----------------------------------------
void ADLParser::next_char() {
+ if (_curchar == '\n') parse_err(WARN, "must call next_line!");
_curchar = *++_ptr;
// if ( _curchar == '\n' ) {
// next_line();
@@ -4515,7 +4744,19 @@ void ADLParser::next_char_or_line() {
//---------------------------next_line-----------------------------------------
void ADLParser::next_line() {
- _curline = _buf.get_line(); _linenum++;
+ _curline = _buf.get_line();
+ _curchar = ' ';
+}
+
+//------------------------get_line_string--------------------------------------
+// Prepended location descriptor, for debugging.
+// Must return a malloced string (that can be freed if desired).
+char* ADLParser::get_line_string(int linenum) {
+ const char* file = _AD._ADL_file._name;
+ int line = linenum ? linenum : this->linenum();
+ char* location = (char *)malloc(strlen(file) + 100);
+ sprintf(location, "\n#line %d \"%s\"\n", line, file);
+ return location;
}
//-------------------------is_literal_constant---------------------------------
@@ -4556,6 +4797,66 @@ bool ADLParser::is_int_token(const char* token, int& intval) {
return true;
}
+static const char* skip_expr_ws(const char* str) {
+ const char * cp = str;
+ while (cp[0]) {
+ if (cp[0] <= ' ') {
+ ++cp;
+ } else if (cp[0] == '#') {
+ ++cp;
+ while (cp[0] == ' ') ++cp;
+ assert(0 == strncmp(cp, "line", 4), "must be a #line directive");
+ const char* eol = strchr(cp, '\n');
+ assert(eol != NULL, "must find end of line");
+ if (eol == NULL) eol = cp + strlen(cp);
+ cp = eol;
+ } else {
+ break;
+ }
+ }
+ return cp;
+}
+
+//-----------------------equivalent_expressions--------------------------------
+bool ADLParser::equivalent_expressions(const char* str1, const char* str2) {
+ if (str1 == str2)
+ return true;
+ else if (str1 == NULL || str2 == NULL)
+ return false;
+ const char* cp1 = str1;
+ const char* cp2 = str2;
+ char in_quote = '\0';
+ while (cp1[0] && cp2[0]) {
+ if (!in_quote) {
+ // skip spaces and/or cpp directives
+ const char* cp1a = skip_expr_ws(cp1);
+ const char* cp2a = skip_expr_ws(cp2);
+ if (cp1a > cp1 && cp2a > cp2) {
+ cp1 = cp1a; cp2 = cp2a;
+ continue;
+ }
+ if (cp1a > cp1 || cp2a > cp2) break; // fail
+ }
+ // match one non-space char
+ if (cp1[0] != cp2[0]) break; // fail
+ char ch = cp1[0];
+ cp1++; cp2++;
+ // watch for quotes
+ if (in_quote && ch == '\\') {
+ if (cp1[0] != cp2[0]) break; // fail
+ if (!cp1[0]) break;
+ cp1++; cp2++;
+ }
+ if (in_quote && ch == in_quote) {
+ in_quote = '\0';
+ } else if (!in_quote && (ch == '"' || ch == '\'')) {
+ in_quote = ch;
+ }
+ }
+ return (!cp1[0] && !cp2[0]);
+}
+
+
//-------------------------------trim------------------------------------------
void ADLParser::trim(char* &token) {
while (*token <= ' ') token++;
diff --git a/hotspot/src/share/vm/adlc/adlparse.hpp b/hotspot/src/share/vm/adlc/adlparse.hpp
index 853a104f7cd..0840bfc758b 100644
--- a/hotspot/src/share/vm/adlc/adlparse.hpp
+++ b/hotspot/src/share/vm/adlc/adlparse.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1997-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,7 +70,6 @@ class ADLParser {
protected:
char *_curline; // Start of current line
char *_ptr; // Pointer into current location in File Buffer
- int _linenum; // Count of line numbers seen so far
char _curchar; // Current character from buffer
FormDict &_globalNames; // Global names
@@ -94,6 +93,7 @@ protected:
void pipe_parse(void); // Parse pipeline section
void definitions_parse(void); // Parse definitions section
void peep_parse(void); // Parse peephole rule definitions
+ void preproc_line(void); // Parse a #line statement
void preproc_define(void); // Parse a #define statement
void preproc_undef(void); // Parse an #undef statement
@@ -160,9 +160,10 @@ protected:
Interface *interface_parse(); // Parse operand interface rule
Interface *mem_interface_parse(); // Parse memory interface rule
Interface *cond_interface_parse(); // Parse conditional interface rule
- char *interface_field_parse();// Parse field contents
+ char *interface_field_parse(const char** format = NULL);// Parse field contents
FormatRule *format_parse(void); // Parse format rule
+ FormatRule *template_parse(void); // Parse format rule
void effect_parse(InstructForm *instr); // Parse effect rule
ExpandRule *expand_parse(InstructForm *instr); // Parse expand rule
RewriteRule *rewrite_parse(void); // Parse rewrite rule
@@ -226,7 +227,7 @@ protected:
void get_effectlist(FormDict &effects, FormDict &operands); // Parse effect-operand pairs
// Return the contents of a parenthesized expression.
// Requires initial '(' and consumes final ')', which is replaced by '\0'.
- char *get_paren_expr(const char *description);
+ char *get_paren_expr(const char *description, bool include_location = false);
// Return expression up to next stop-char, which terminator replaces.
// Does not require initial '('. Does not consume final stop-char.
// Final stop-char is left in _curchar, but is also is replaced by '\0'.
@@ -234,6 +235,11 @@ protected:
char *find_cpp_block(const char *description); // Parse a C++ code block
// Issue parser error message & go to EOL
void parse_err(int flag, const char *fmt, ...);
+ // Create a location marker for this file and line.
+ char *get_line_string(int linenum = 0);
+ // Return a location marker which tells the C preprocessor to
+ // forget the previous location marker. (Requires awk postprocessing.)
+ char *end_line_marker() { return (char*)"\n#line 999999\n"; }
// Return pointer to current character
inline char cur_char(void);
@@ -263,10 +269,11 @@ public:
void parse(void); // Do the parsing & build forms lists
- int getlines( ) { return _linenum; }
+ int linenum() { return _buf.linenum(); }
static bool is_literal_constant(const char *hex_string);
static bool is_hex_digit(char digit);
static bool is_int_token(const char* token, int& intval);
+ static bool equivalent_expressions(const char* str1, const char* str2);
static void trim(char* &token); // trim leading & trailing spaces
};
diff --git a/hotspot/src/share/vm/adlc/archDesc.cpp b/hotspot/src/share/vm/adlc/archDesc.cpp
index 0aa15064c5b..a73ad76da23 100644
--- a/hotspot/src/share/vm/adlc/archDesc.cpp
+++ b/hotspot/src/share/vm/adlc/archDesc.cpp
@@ -140,7 +140,7 @@ bool MatchList::search(const char *opc, const char *res, const char *lch,
if ((rch == _rchild) || (rch && _rchild && !strcmp(rch, _rchild))) {
char * predStr = get_pred();
char * prStr = pr?pr->_pred:NULL;
- if ((prStr == predStr) || (prStr && predStr && !strcmp(prStr, predStr))) {
+ if (ADLParser::equivalent_expressions(prStr, predStr)) {
return true;
}
}
@@ -212,9 +212,9 @@ ArchDesc::ArchDesc()
// Initialize I/O Files
_ADL_file._name = NULL; _ADL_file._fp = NULL;
// Machine dependent output files
- _DFA_file._name = "dfa_i486.cpp"; _DFA_file._fp = NULL;
- _HPP_file._name = "ad_i486.hpp"; _HPP_file._fp = NULL;
- _CPP_file._name = "ad_i486.cpp"; _CPP_file._fp = NULL;
+ _DFA_file._name = NULL; _DFA_file._fp = NULL;
+ _HPP_file._name = NULL; _HPP_file._fp = NULL;
+ _CPP_file._name = NULL; _CPP_file._fp = NULL;
_bug_file._name = "bugs.out"; _bug_file._fp = NULL;
// Initialize Register & Pipeline Form Pointers
diff --git a/hotspot/src/share/vm/adlc/dfa.cpp b/hotspot/src/share/vm/adlc/dfa.cpp
index 36d56062cfd..1075c9da774 100644
--- a/hotspot/src/share/vm/adlc/dfa.cpp
+++ b/hotspot/src/share/vm/adlc/dfa.cpp
@@ -458,7 +458,7 @@ void ArchDesc::buildDFA(FILE* fp) {
class dfa_shared_preds {
- enum { count = 2 };
+ enum { count = 4 };
static bool _found[count];
static const char* _type [count];
@@ -479,12 +479,15 @@ class dfa_shared_preds {
char c = *prev;
switch( c ) {
case ' ':
+ case '\n':
return dfa_shared_preds::valid_loc(pred, prev);
case '!':
case '(':
case '<':
case '=':
return true;
+ case '"': // such as: #line 10 "myfile.ad"\n mypredicate
+ return true;
case '|':
if( prev != pred && *(prev-1) == '|' ) return true;
case '&':
@@ -564,10 +567,14 @@ public:
}
};
// shared predicates, _var and _pred entry should be the same length
-bool dfa_shared_preds::_found[dfa_shared_preds::count] = { false, false };
-const char* dfa_shared_preds::_type[dfa_shared_preds::count] = { "int", "bool" };
-const char* dfa_shared_preds::_var [dfa_shared_preds::count] = { "_n_get_int__", "Compile__current____select_24_bit_instr__" };
-const char* dfa_shared_preds::_pred[dfa_shared_preds::count] = { "n->get_int()", "Compile::current()->select_24_bit_instr()" };
+bool dfa_shared_preds::_found[dfa_shared_preds::count]
+ = { false, false, false, false };
+const char* dfa_shared_preds::_type[dfa_shared_preds::count]
+ = { "int", "jlong", "intptr_t", "bool" };
+const char* dfa_shared_preds::_var [dfa_shared_preds::count]
+ = { "_n_get_int__", "_n_get_long__", "_n_get_intptr_t__", "Compile__current____select_24_bit_instr__" };
+const char* dfa_shared_preds::_pred[dfa_shared_preds::count]
+ = { "n->get_int()", "n->get_long()", "n->get_intptr_t()", "Compile::current()->select_24_bit_instr()" };
void ArchDesc::gen_dfa_state_body(FILE* fp, Dict &minimize, ProductionState &status, Dict &operands_chained_from, int i) {
diff --git a/hotspot/src/share/vm/adlc/filebuff.cpp b/hotspot/src/share/vm/adlc/filebuff.cpp
index 502354edcf0..9cccb5d069e 100644
--- a/hotspot/src/share/vm/adlc/filebuff.cpp
+++ b/hotspot/src/share/vm/adlc/filebuff.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1997-2002 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,7 @@ FileBuff::FileBuff( BufferedFile *fptr, ArchDesc& archDesc) : _fp(fptr), _AD(arc
exit(1); // Exit on seek error
}
_filepos = ftell(_fp->_fp); // Reset current file position
+ _linenum = 0;
_bigbuf = new char[_bufferSize]; // Create buffer to hold text for parser
if( !_bigbuf ) {
@@ -76,6 +77,7 @@ char *FileBuff::get_line(void) {
// Check for end of file & return NULL
if (_bufeol >= _bufmax) return NULL;
+ _linenum++;
retval = ++_bufeol; // return character following end of previous line
if (*retval == '\0') return NULL; // Check for EOF sentinal
// Search for newline character which must end each line
diff --git a/hotspot/src/share/vm/adlc/filebuff.hpp b/hotspot/src/share/vm/adlc/filebuff.hpp
index fdfa985db4c..8f12f9262d9 100644
--- a/hotspot/src/share/vm/adlc/filebuff.hpp
+++ b/hotspot/src/share/vm/adlc/filebuff.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1997-2004 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -51,6 +51,7 @@ class FileBuff {
int _err; // Error flag for file seek/read operations
long _filepos; // Current offset from start of file
+ int _linenum;
ArchDesc& _AD; // Reference to Architecture Description
@@ -66,6 +67,8 @@ class FileBuff {
// This returns a pointer to the start of the current line in the buffer,
// and increments bufeol and filepos to point at the end of that line.
char *get_line(void);
+ int linenum() const { return _linenum; }
+ void set_linenum(int line) { _linenum = line; }
// This converts a pointer into the buffer to a file offset. It only works
// when the pointer is valid (i.e. just obtained from getline()).
diff --git a/hotspot/src/share/vm/adlc/forms.cpp b/hotspot/src/share/vm/adlc/forms.cpp
index e9d7de211ea..e2083c68259 100644
--- a/hotspot/src/share/vm/adlc/forms.cpp
+++ b/hotspot/src/share/vm/adlc/forms.cpp
@@ -35,6 +35,8 @@ Arena *Form::generate_arena() {
//------------------------------NameList---------------------------------------
// reserved user-defined string
const char *NameList::_signal = "$$SIGNAL$$";
+const char *NameList::_signal2 = "$$SIGNAL2$$";
+const char *NameList::_signal3 = "$$SIGNAL3$$";
// Constructor and Destructor
NameList::NameList() : _cur(0), _max(4), _iter(0), _justReset(true) {
diff --git a/hotspot/src/share/vm/adlc/forms.hpp b/hotspot/src/share/vm/adlc/forms.hpp
index c510ac43aef..bf930b8e6da 100644
--- a/hotspot/src/share/vm/adlc/forms.hpp
+++ b/hotspot/src/share/vm/adlc/forms.hpp
@@ -329,6 +329,8 @@ protected:
public:
static const char *_signal; // reserved user-defined string
+ static const char *_signal2; // reserved user-defined string
+ static const char *_signal3; // reserved user-defined string
enum { Not_in_list = -1 };
void addName(const char *name);
diff --git a/hotspot/src/share/vm/adlc/formssel.cpp b/hotspot/src/share/vm/adlc/formssel.cpp
index b45de088afb..65b511bcdb1 100644
--- a/hotspot/src/share/vm/adlc/formssel.cpp
+++ b/hotspot/src/share/vm/adlc/formssel.cpp
@@ -1102,10 +1102,7 @@ bool equivalent_predicates( const InstructForm *instr1, const InstructForm *inst
}
if( pred1 != NULL && pred2 != NULL ) {
// compare the predicates
- const char *str1 = pred1->_pred;
- const char *str2 = pred2->_pred;
- if( (str1 == NULL && str2 == NULL)
- || (str1 != NULL && str2 != NULL && strcmp(str1,str2) == 0) ) {
+ if (ADLParser::equivalent_expressions(pred1->_pred, pred2->_pred)) {
return true;
}
}
@@ -1574,10 +1571,10 @@ Opcode::opcode_type Opcode::as_opcode_type(const char *param) {
return Opcode::NOT_AN_OPCODE;
}
-void Opcode::print_opcode(FILE *fp, Opcode::opcode_type desired_opcode) {
+bool Opcode::print_opcode(FILE *fp, Opcode::opcode_type desired_opcode) {
// Default values previously provided by MachNode::primary()...
- const char *description = "default_opcode()";
- const char *value = "-1";
+ const char *description = NULL;
+ const char *value = NULL;
// Check if user provided any opcode definitions
if( this != NULL ) {
// Update 'value' if user provided a definition in the instruction
@@ -1599,7 +1596,10 @@ void Opcode::print_opcode(FILE *fp, Opcode::opcode_type desired_opcode) {
break;
}
}
- fprintf(fp, "(%s /*%s*/)", value, description);
+ if (value != NULL) {
+ fprintf(fp, "(%s /*%s*/)", value, description);
+ }
+ return value != NULL;
}
void Opcode::dump() {
@@ -2610,14 +2610,19 @@ void MemInterface::output(FILE *fp) {
}
//------------------------------CondInterface----------------------------------
-CondInterface::CondInterface(char *equal, char *not_equal,
- char *less, char *greater_equal,
- char *less_equal, char *greater)
+CondInterface::CondInterface(const char* equal, const char* equal_format,
+ const char* not_equal, const char* not_equal_format,
+ const char* less, const char* less_format,
+ const char* greater_equal, const char* greater_equal_format,
+ const char* less_equal, const char* less_equal_format,
+ const char* greater, const char* greater_format)
: Interface("COND_INTER"),
- _equal(equal), _not_equal(not_equal),
- _less(less), _greater_equal(greater_equal),
- _less_equal(less_equal), _greater(greater) {
- //
+ _equal(equal), _equal_format(equal_format),
+ _not_equal(not_equal), _not_equal_format(not_equal_format),
+ _less(less), _less_format(less_format),
+ _greater_equal(greater_equal), _greater_equal_format(greater_equal_format),
+ _less_equal(less_equal), _less_equal_format(less_equal_format),
+ _greater(greater), _greater_format(greater_format) {
}
CondInterface::~CondInterface() {
// not owner of any character arrays
@@ -3316,7 +3321,7 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
"Load8B" ,"Load4B" ,"Load8C" ,"Load4C" ,"Load2C" ,"Load8S", "Load4S","Load2S",
"LoadRange", "LoadKlass", "LoadNKlass", "LoadL_unaligned", "LoadD_unaligned",
"LoadPLocked", "LoadLLocked",
- "StorePConditional", "StoreLConditional",
+ "StorePConditional", "StoreIConditional", "StoreLConditional",
"CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN",
"StoreCM",
"ClearArray"
@@ -3768,6 +3773,10 @@ bool MatchRule::is_chain_rule(FormDict &globals) const {
int MatchRule::is_ideal_copy() const {
if( _rChild ) {
const char *opType = _rChild->_opType;
+#if 1
+ if( strcmp(opType,"CastIP")==0 )
+ return 1;
+#else
if( strcmp(opType,"CastII")==0 )
return 1;
// Do not treat *CastPP this way, because it
@@ -3787,6 +3796,7 @@ int MatchRule::is_ideal_copy() const {
// return 1;
//if( strcmp(opType,"CastP2X")==0 )
// return 1;
+#endif
}
if( is_chain_rule(_AD.globalNames()) &&
_lChild && strncmp(_lChild->_opType,"stackSlot",9)==0 )
diff --git a/hotspot/src/share/vm/adlc/formssel.hpp b/hotspot/src/share/vm/adlc/formssel.hpp
index 8ed2885c890..a363f16ee41 100644
--- a/hotspot/src/share/vm/adlc/formssel.hpp
+++ b/hotspot/src/share/vm/adlc/formssel.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1998-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1998-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -397,7 +397,7 @@ public:
void output(FILE *fp);
// --------------------------- FILE *output_routines
- void print_opcode(FILE *fp, Opcode::opcode_type desired_opcode);
+ bool print_opcode(FILE *fp, Opcode::opcode_type desired_opcode);
};
//------------------------------InsEncode--------------------------------------
@@ -779,10 +779,20 @@ public:
const char *_greater_equal;
const char *_less_equal;
const char *_greater;
+ const char *_equal_format;
+ const char *_not_equal_format;
+ const char *_less_format;
+ const char *_greater_equal_format;
+ const char *_less_equal_format;
+ const char *_greater_format;
// Public Methods
- CondInterface(char *equal, char *not_equal, char *less, char *greater_equal,
- char *less_equal, char *greater);
+ CondInterface(const char* equal, const char* equal_format,
+ const char* not_equal, const char* not_equal_format,
+ const char* less, const char* less_format,
+ const char* greater_equal, const char* greater_equal_format,
+ const char* less_equal, const char* less_equal_format,
+ const char* greater, const char* greater_format);
~CondInterface();
void dump();
diff --git a/hotspot/src/share/vm/adlc/output_c.cpp b/hotspot/src/share/vm/adlc/output_c.cpp
index eb33e67b376..dbc287b1d1a 100644
--- a/hotspot/src/share/vm/adlc/output_c.cpp
+++ b/hotspot/src/share/vm/adlc/output_c.cpp
@@ -1619,6 +1619,7 @@ void ArchDesc::defineExpand(FILE *fp, InstructForm *node) {
}
// Iterate over the new instruction's operands
+ int prev_pos = -1;
for( expand_instr->reset(); (opid = expand_instr->iter()) != NULL; ) {
// Use 'parameter' at current position in list of new instruction's formals
// instead of 'opid' when looking up info internal to new_inst
@@ -1642,6 +1643,18 @@ void ArchDesc::defineExpand(FILE *fp, InstructForm *node) {
// ins = (InstructForm *) _globalNames[new_id];
exp_pos = node->operand_position_format(opid);
assert(exp_pos != -1, "Bad expand rule");
+ if (prev_pos > exp_pos && expand_instruction->_matrule != NULL) {
+ // For the add_req calls below to work correctly they need
+ // to added in the same order that a match would add them.
+ // This means that they would need to be in the order of
+ // the components list instead of the formal parameters.
+ // This is a sort of hidden invariant that previously
+ // wasn't checked and could lead to incorrectly
+ // constructed nodes.
+ syntax_err(node->_linenum, "For expand in %s to work, parameter declaration order in %s must follow matchrule\n",
+ node->_ident, new_inst->_ident);
+ }
+ prev_pos = exp_pos;
new_pos = new_inst->operand_position(parameter,Component::USE);
if (new_pos != -1) {
@@ -2306,7 +2319,12 @@ private:
_processing_noninput = false;
// A replacement variable, originally '$'
if ( Opcode::as_opcode_type(rep_var) != Opcode::NOT_AN_OPCODE ) {
- _inst._opcode->print_opcode(_fp, Opcode::as_opcode_type(rep_var) );
+ if (!_inst._opcode->print_opcode(_fp, Opcode::as_opcode_type(rep_var) )) {
+ // Missing opcode
+ _AD.syntax_err( _inst._linenum,
+ "Missing $%s opcode definition in %s, used by encoding %s\n",
+ rep_var, _inst._ident, _encoding._name);
+ }
}
else {
// Lookup its position in parameter list
@@ -2348,7 +2366,13 @@ private:
else if( Opcode::as_opcode_type(inst_rep_var) != Opcode::NOT_AN_OPCODE ) {
// else check if "primary", "secondary", "tertiary"
assert( _constant_status == LITERAL_ACCESSED, "Must be processing a literal constant parameter");
- _inst._opcode->print_opcode(_fp, Opcode::as_opcode_type(inst_rep_var) );
+ if (!_inst._opcode->print_opcode(_fp, Opcode::as_opcode_type(inst_rep_var) )) {
+ // Missing opcode
+ _AD.syntax_err( _inst._linenum,
+ "Missing $%s opcode definition in %s\n",
+ rep_var, _inst._ident);
+
+ }
_constant_status = LITERAL_OUTPUT;
}
else if((_AD.get_registers() != NULL ) && (_AD.get_registers()->getRegDef(inst_rep_var) != NULL)) {
diff --git a/hotspot/src/share/vm/adlc/output_h.cpp b/hotspot/src/share/vm/adlc/output_h.cpp
index 04cddcd28c8..d2fd489abe9 100644
--- a/hotspot/src/share/vm/adlc/output_h.cpp
+++ b/hotspot/src/share/vm/adlc/output_h.cpp
@@ -355,17 +355,19 @@ static void defineConstructor(FILE *fp, const char *name, uint num_consts,
// ---------------------------------------------------------------------------
// Generate the format rule for condition codes
-static void defineCCodeDump(FILE *fp, int i) {
- fprintf(fp, " if( _c%d == BoolTest::eq ) st->print(\"eq\");\n",i);
- fprintf(fp, " else if( _c%d == BoolTest::ne ) st->print(\"ne\");\n",i);
- fprintf(fp, " else if( _c%d == BoolTest::le ) st->print(\"le\");\n",i);
- fprintf(fp, " else if( _c%d == BoolTest::ge ) st->print(\"ge\");\n",i);
- fprintf(fp, " else if( _c%d == BoolTest::lt ) st->print(\"lt\");\n",i);
- fprintf(fp, " else if( _c%d == BoolTest::gt ) st->print(\"gt\");\n",i);
+static void defineCCodeDump(OperandForm* oper, FILE *fp, int i) {
+ assert(oper != NULL, "what");
+ CondInterface* cond = oper->_interface->is_CondInterface();
+ fprintf(fp, " if( _c%d == BoolTest::eq ) st->print(\"%s\");\n",i,cond->_equal_format);
+ fprintf(fp, " else if( _c%d == BoolTest::ne ) st->print(\"%s\");\n",i,cond->_not_equal_format);
+ fprintf(fp, " else if( _c%d == BoolTest::le ) st->print(\"%s\");\n",i,cond->_less_equal_format);
+ fprintf(fp, " else if( _c%d == BoolTest::ge ) st->print(\"%s\");\n",i,cond->_greater_equal_format);
+ fprintf(fp, " else if( _c%d == BoolTest::lt ) st->print(\"%s\");\n",i,cond->_less_format);
+ fprintf(fp, " else if( _c%d == BoolTest::gt ) st->print(\"%s\");\n",i,cond->_greater_format);
}
// Output code that dumps constant values, increment "i" if type is constant
-static uint dump_spec_constant(FILE *fp, const char *ideal_type, uint i) {
+static uint dump_spec_constant(FILE *fp, const char *ideal_type, uint i, OperandForm* oper) {
if (!strcmp(ideal_type, "ConI")) {
fprintf(fp," st->print(\"#%%d\", _c%d);\n", i);
++i;
@@ -375,7 +377,7 @@ static uint dump_spec_constant(FILE *fp, const char *ideal_type, uint i) {
++i;
}
else if (!strcmp(ideal_type, "ConN")) {
- fprintf(fp," _c%d->dump();\n", i);
+ fprintf(fp," _c%d->dump_on(st);\n", i);
++i;
}
else if (!strcmp(ideal_type, "ConL")) {
@@ -391,7 +393,7 @@ static uint dump_spec_constant(FILE *fp, const char *ideal_type, uint i) {
++i;
}
else if (!strcmp(ideal_type, "Bool")) {
- defineCCodeDump(fp,i);
+ defineCCodeDump(oper, fp,i);
++i;
}
@@ -476,7 +478,7 @@ void gen_oper_format(FILE *fp, FormDict &globals, OperandForm &oper, bool for_c_
}
// ALWAYS! Provide a special case output for condition codes.
if( oper.is_ideal_bool() ) {
- defineCCodeDump(fp,0);
+ defineCCodeDump(&oper, fp,0);
}
fprintf(fp,"}\n");
@@ -549,7 +551,7 @@ void gen_oper_format(FILE *fp, FormDict &globals, OperandForm &oper, bool for_c_
}
// ALWAYS! Provide a special case output for condition codes.
if( oper.is_ideal_bool() ) {
- defineCCodeDump(fp,0);
+ defineCCodeDump(&oper, fp,0);
}
fprintf(fp, "}\n");
fprintf(fp, "#endif\n");
@@ -583,10 +585,53 @@ void gen_inst_format(FILE *fp, FormDict &globals, InstructForm &inst, bool for_c
while( (string = inst._format->_strings.iter()) != NULL ) {
fprintf(fp," ");
// Check if this is a standard string or a replacement variable
- if( string != NameList::_signal ) // Normal string. Pass through.
+ if( string == NameList::_signal ) { // Replacement variable
+ const char* rep_var = inst._format->_rep_vars.iter();
+ inst.rep_var_format( fp, rep_var);
+ } else if( string == NameList::_signal3 ) { // Replacement variable in raw text
+ const char* rep_var = inst._format->_rep_vars.iter();
+ const Form *form = inst._localNames[rep_var];
+ if (form == NULL) {
+ fprintf(stderr, "unknown replacement variable in format statement: '%s'\n", rep_var);
+ assert(false, "ShouldNotReachHere()");
+ }
+ OpClassForm *opc = form->is_opclass();
+ assert( opc, "replacement variable was not found in local names");
+ // Lookup the index position of the replacement variable
+ int idx = inst.operand_position_format(rep_var);
+ if ( idx == -1 ) {
+ assert( strcmp(opc->_ident,"label")==0, "Unimplemented");
+ assert( false, "ShouldNotReachHere()");
+ }
+
+ if (inst.is_noninput_operand(idx)) {
+ assert( false, "ShouldNotReachHere()");
+ } else {
+ // Output the format call for this operand
+ fprintf(fp,"opnd_array(%d)",idx);
+ }
+ rep_var = inst._format->_rep_vars.iter();
+ inst._format->_strings.iter();
+ if ( strcmp(rep_var,"$constant") == 0 && opc->is_operand()) {
+ Form::DataType constant_type = form->is_operand()->is_base_constant(globals);
+ if ( constant_type == Form::idealD ) {
+ fprintf(fp,"->constantD()");
+ } else if ( constant_type == Form::idealF ) {
+ fprintf(fp,"->constantF()");
+ } else if ( constant_type == Form::idealL ) {
+ fprintf(fp,"->constantL()");
+ } else {
+ fprintf(fp,"->constant()");
+ }
+ } else if ( strcmp(rep_var,"$cmpcode") == 0) {
+ fprintf(fp,"->ccode()");
+ } else {
+ assert( false, "ShouldNotReachHere()");
+ }
+ } else if( string == NameList::_signal2 ) // Raw program text
+ fputs(inst._format->_strings.iter(), fp);
+ else
fprintf(fp,"st->print(\"%s\");\n", string);
- else // Replacement variable
- inst.rep_var_format( fp, inst._format->_rep_vars.iter() );
} // Done with all format strings
} // Done generating the user-defined portion of the format
@@ -1404,7 +1449,7 @@ void ArchDesc::declareClasses(FILE *fp) {
oper->_components.reset();
if ((comp = oper->_components.iter()) == NULL) {
assert(num_consts == 1, "Bad component list detected.\n");
- i = dump_spec_constant( fp, type, i );
+ i = dump_spec_constant( fp, type, i, oper );
// Check that type actually matched
assert( i != 0, "Non-constant operand lacks component list.");
} // end if NULL
@@ -1414,7 +1459,7 @@ void ArchDesc::declareClasses(FILE *fp) {
oper->_components.reset();
while((comp = oper->_components.iter()) != NULL) {
type = comp->base_type(_globalNames);
- i = dump_spec_constant( fp, type, i );
+ i = dump_spec_constant( fp, type, i, NULL );
}
}
// finish line (3)
diff --git a/hotspot/src/share/vm/asm/assembler.cpp b/hotspot/src/share/vm/asm/assembler.cpp
index 34ae81ce1cd..e9d99621b6b 100644
--- a/hotspot/src/share/vm/asm/assembler.cpp
+++ b/hotspot/src/share/vm/asm/assembler.cpp
@@ -249,8 +249,6 @@ void AbstractAssembler::block_comment(const char* comment) {
bool MacroAssembler::needs_explicit_null_check(intptr_t offset) {
// Exception handler checks the nmethod's implicit null checks table
// only when this method returns false.
-#ifndef SPARC
- // Sparc does not have based addressing
if (UseCompressedOops) {
// The first page after heap_base is unmapped and
// the 'offset' is equal to [heap_base + offset] for
@@ -261,7 +259,6 @@ bool MacroAssembler::needs_explicit_null_check(intptr_t offset) {
offset = (intptr_t)(pointer_delta((void*)offset, (void*)heap_base, 1));
}
}
-#endif // SPARC
return offset < 0 || os::vm_page_size() <= offset;
}
diff --git a/hotspot/src/share/vm/c1/c1_CodeStubs.hpp b/hotspot/src/share/vm/c1/c1_CodeStubs.hpp
index 9009e0d8c72..4c47e777b2b 100644
--- a/hotspot/src/share/vm/c1/c1_CodeStubs.hpp
+++ b/hotspot/src/share/vm/c1/c1_CodeStubs.hpp
@@ -482,3 +482,81 @@ class ArrayCopyStub: public CodeStub {
virtual void print_name(outputStream* out) const { out->print("ArrayCopyStub"); }
#endif // PRODUCT
};
+
+//////////////////////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+// Code stubs for Garbage-First barriers.
+class G1PreBarrierStub: public CodeStub {
+ private:
+ LIR_Opr _addr;
+ LIR_Opr _pre_val;
+ LIR_PatchCode _patch_code;
+ CodeEmitInfo* _info;
+
+ public:
+ // pre_val (a temporary register) must be a register;
+ // addr (the address of the field to be read) must be a LIR_Address
+ G1PreBarrierStub(LIR_Opr addr, LIR_Opr pre_val, LIR_PatchCode patch_code, CodeEmitInfo* info) :
+ _addr(addr), _pre_val(pre_val), _patch_code(patch_code), _info(info)
+ {
+ assert(_pre_val->is_register(), "should be temporary register");
+ assert(_addr->is_address(), "should be the address of the field");
+ }
+
+ LIR_Opr addr() const { return _addr; }
+ LIR_Opr pre_val() const { return _pre_val; }
+ LIR_PatchCode patch_code() const { return _patch_code; }
+ CodeEmitInfo* info() const { return _info; }
+
+ virtual void emit_code(LIR_Assembler* e);
+ virtual void visit(LIR_OpVisitState* visitor) {
+ // don't pass in the code emit info since it's processed in the fast
+ // path
+ if (_info != NULL)
+ visitor->do_slow_case(_info);
+ else
+ visitor->do_slow_case();
+ visitor->do_input(_addr);
+ visitor->do_temp(_pre_val);
+ }
+#ifndef PRODUCT
+ virtual void print_name(outputStream* out) const { out->print("G1PreBarrierStub"); }
+#endif // PRODUCT
+};
+
+class G1PostBarrierStub: public CodeStub {
+ private:
+ LIR_Opr _addr;
+ LIR_Opr _new_val;
+
+ static jbyte* _byte_map_base;
+ static jbyte* byte_map_base_slow();
+ static jbyte* byte_map_base() {
+ if (_byte_map_base == NULL) {
+ _byte_map_base = byte_map_base_slow();
+ }
+ return _byte_map_base;
+ }
+
+ public:
+ // addr (the address of the object head) and new_val must be registers.
+ G1PostBarrierStub(LIR_Opr addr, LIR_Opr new_val): _addr(addr), _new_val(new_val) { }
+
+ LIR_Opr addr() const { return _addr; }
+ LIR_Opr new_val() const { return _new_val; }
+
+ virtual void emit_code(LIR_Assembler* e);
+ virtual void visit(LIR_OpVisitState* visitor) {
+ // don't pass in the code emit info since it's processed in the fast path
+ visitor->do_slow_case();
+ visitor->do_input(_addr);
+ visitor->do_input(_new_val);
+ }
+#ifndef PRODUCT
+ virtual void print_name(outputStream* out) const { out->print("G1PostBarrierStub"); }
+#endif // PRODUCT
+};
+
+#endif // SERIALGC
+//////////////////////////////////////////////////////////////////////////////////////////
diff --git a/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp b/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp
index efa25e3915f..5b7834c8b34 100644
--- a/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp
+++ b/hotspot/src/share/vm/c1/c1_GraphBuilder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1999-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1999-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -676,21 +676,6 @@ GraphBuilder::ScopeData::ScopeData(ScopeData* parent)
}
-void GraphBuilder::kill_field(ciField* field) {
- if (UseLocalValueNumbering) {
- vmap()->kill_field(field);
- }
-}
-
-
-void GraphBuilder::kill_array(Value value) {
- if (UseLocalValueNumbering) {
- vmap()->kill_array(value->type());
- }
- _memory->store_value(value);
-}
-
-
void GraphBuilder::kill_all() {
if (UseLocalValueNumbering) {
vmap()->kill_all();
@@ -987,8 +972,8 @@ void GraphBuilder::store_indexed(BasicType type) {
length = append(new ArrayLength(array, lock_stack()));
}
StoreIndexed* result = new StoreIndexed(array, index, length, type, value, lock_stack());
- kill_array(value); // invalidate all CSEs that are memory accesses of the same type
append(result);
+ _memory->store_value(value);
}
@@ -1478,9 +1463,6 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
case Bytecodes::_putstatic:
{ Value val = pop(type);
append(new StoreField(append(obj), offset, field, val, true, lock_stack(), state_copy, is_loaded, is_initialized));
- if (UseLocalValueNumbering) {
- vmap()->kill_field(field); // invalidate all CSEs that are memory accesses
- }
}
break;
case Bytecodes::_getfield :
@@ -1503,7 +1485,6 @@ void GraphBuilder::access_field(Bytecodes::Code code) {
if (is_loaded) store = _memory->store(store);
if (store != NULL) {
append(store);
- kill_field(field); // invalidate all CSEs that are accesses of this field
}
}
break;
@@ -1900,6 +1881,8 @@ Instruction* GraphBuilder::append_with_bci(Instruction* instr, int bci) {
assert(i2->bci() != -1, "should already be linked");
return i2;
}
+ ValueNumberingEffects vne(vmap());
+ i1->visit(&vne);
}
if (i1->as_Phi() == NULL && i1->as_Local() == NULL) {
@@ -1926,14 +1909,8 @@ Instruction* GraphBuilder::append_with_bci(Instruction* instr, int bci) {
assert(_last == i1, "adjust code below");
StateSplit* s = i1->as_StateSplit();
if (s != NULL && i1->as_BlockEnd() == NULL) {
- // Continue CSE across certain intrinsics
- Intrinsic* intrinsic = s->as_Intrinsic();
- if (UseLocalValueNumbering) {
- if (intrinsic == NULL || !intrinsic->preserves_state()) {
- vmap()->kill_all(); // for now, hopefully we need this only for calls eventually
- }
- }
if (EliminateFieldAccess) {
+ Intrinsic* intrinsic = s->as_Intrinsic();
if (s->as_Invoke() != NULL || (intrinsic && !intrinsic->preserves_state())) {
_memory->kill();
}
diff --git a/hotspot/src/share/vm/c1/c1_GraphBuilder.hpp b/hotspot/src/share/vm/c1/c1_GraphBuilder.hpp
index 46786748b5e..aff61c6cac2 100644
--- a/hotspot/src/share/vm/c1/c1_GraphBuilder.hpp
+++ b/hotspot/src/share/vm/c1/c1_GraphBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1999-2006 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1999-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -283,8 +283,6 @@ class GraphBuilder VALUE_OBJ_CLASS_SPEC {
Dependencies* dependency_recorder() const; // = compilation()->dependencies()
bool direct_compare(ciKlass* k);
- void kill_field(ciField* field);
- void kill_array(Value value);
void kill_all();
ValueStack* lock_stack();
diff --git a/hotspot/src/share/vm/c1/c1_IR.cpp b/hotspot/src/share/vm/c1/c1_IR.cpp
index 005114b46e2..f1a268de40e 100644
--- a/hotspot/src/share/vm/c1/c1_IR.cpp
+++ b/hotspot/src/share/vm/c1/c1_IR.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1999-2006 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1999-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -574,12 +574,23 @@ void ComputeLinearScanOrder::count_edges(BlockBegin* cur, BlockBegin* parent) {
TRACE_LINEAR_SCAN(3, tty->print_cr("backward branch"));
assert(is_visited(cur), "block must be visisted when block is active");
assert(parent != NULL, "must have parent");
- assert(parent->number_of_sux() == 1, "loop end blocks must have one successor (critical edges are split)");
cur->set(BlockBegin::linear_scan_loop_header_flag);
cur->set(BlockBegin::backward_branch_target_flag);
parent->set(BlockBegin::linear_scan_loop_end_flag);
+
+ // When a loop header is also the start of an exception handler, then the backward branch is
+ // an exception edge. Because such edges are usually critical edges which cannot be split, the
+ // loop must be excluded here from processing.
+ if (cur->is_set(BlockBegin::exception_entry_flag)) {
+ // Make sure that dominators are correct in this weird situation
+ _iterative_dominators = true;
+ return;
+ }
+ assert(parent->number_of_sux() == 1 && parent->sux_at(0) == cur,
+ "loop end blocks must have one successor (critical edges are split)");
+
_loop_end_blocks.append(parent);
return;
}
diff --git a/hotspot/src/share/vm/c1/c1_LIRAssembler.cpp b/hotspot/src/share/vm/c1/c1_LIRAssembler.cpp
index 48f76fd7767..5a46b03e5d8 100644
--- a/hotspot/src/share/vm/c1/c1_LIRAssembler.cpp
+++ b/hotspot/src/share/vm/c1/c1_LIRAssembler.cpp
@@ -74,6 +74,7 @@ void LIR_Assembler::patching_epilog(PatchingStub* patch, LIR_PatchCode patch_cod
LIR_Assembler::LIR_Assembler(Compilation* c):
_compilation(c)
, _masm(c->masm())
+ , _bs(Universe::heap()->barrier_set())
, _frame_map(c->frame_map())
, _current_block(NULL)
, _pending_non_safepoint(NULL)
diff --git a/hotspot/src/share/vm/c1/c1_LIRAssembler.hpp b/hotspot/src/share/vm/c1/c1_LIRAssembler.hpp
index 1cbf564288c..f5de2c73479 100644
--- a/hotspot/src/share/vm/c1/c1_LIRAssembler.hpp
+++ b/hotspot/src/share/vm/c1/c1_LIRAssembler.hpp
@@ -24,11 +24,13 @@
class Compilation;
class ScopeValue;
+class BarrierSet;
class LIR_Assembler: public CompilationResourceObj {
private:
C1_MacroAssembler* _masm;
CodeStubList* _slow_case_stubs;
+ BarrierSet* _bs;
Compilation* _compilation;
FrameMap* _frame_map;
diff --git a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
index 209899357d3..9b718a3d28f 100644
--- a/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
+++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.cpp
@@ -285,16 +285,7 @@ jlong LIRItem::get_jlong_constant() const {
void LIRGenerator::init() {
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
- CardTableModRefBS* ct = (CardTableModRefBS*)bs;
- assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
-
-#ifdef _LP64
- _card_table_base = new LIR_Const((jlong)ct->byte_map_base);
-#else
- _card_table_base = new LIR_Const((jint)ct->byte_map_base);
-#endif
+ _bs = Universe::heap()->barrier_set();
}
@@ -1219,8 +1210,8 @@ LIR_Opr LIRGenerator::load_constant(LIR_Const* c) {
break;
case T_LONG:
case T_DOUBLE:
- if (c->as_jint_hi_bits() != other->as_jint_lo_bits()) continue;
- if (c->as_jint_lo_bits() != other->as_jint_hi_bits()) continue;
+ if (c->as_jint_hi_bits() != other->as_jint_hi_bits()) continue;
+ if (c->as_jint_lo_bits() != other->as_jint_lo_bits()) continue;
break;
case T_OBJECT:
if (c->as_jobject() != other->as_jobject()) continue;
@@ -1239,8 +1230,37 @@ LIR_Opr LIRGenerator::load_constant(LIR_Const* c) {
// Various barriers
+void LIRGenerator::pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info) {
+ // Do the pre-write barrier, if any.
+ switch (_bs->kind()) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ G1SATBCardTableModRef_pre_barrier(addr_opr, patch, info);
+ break;
+#endif // SERIALGC
+ case BarrierSet::CardTableModRef:
+ case BarrierSet::CardTableExtension:
+ // No pre barriers
+ break;
+ case BarrierSet::ModRef:
+ case BarrierSet::Other:
+ // No pre barriers
+ break;
+ default :
+ ShouldNotReachHere();
+
+ }
+}
+
void LIRGenerator::post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
- switch (Universe::heap()->barrier_set()->kind()) {
+ switch (_bs->kind()) {
+#ifndef SERIALGC
+ case BarrierSet::G1SATBCT:
+ case BarrierSet::G1SATBCTLogging:
+ G1SATBCardTableModRef_post_barrier(addr, new_val);
+ break;
+#endif // SERIALGC
case BarrierSet::CardTableModRef:
case BarrierSet::CardTableExtension:
CardTableModRef_post_barrier(addr, new_val);
@@ -1254,11 +1274,120 @@ void LIRGenerator::post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
}
}
+////////////////////////////////////////////////////////////////////////
+#ifndef SERIALGC
+
+void LIRGenerator::G1SATBCardTableModRef_pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info) {
+ if (G1DisablePreBarrier) return;
+
+ // First we test whether marking is in progress.
+ BasicType flag_type;
+ if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
+ flag_type = T_INT;
+ } else {
+ guarantee(in_bytes(PtrQueue::byte_width_of_active()) == 1,
+ "Assumption");
+ flag_type = T_BYTE;
+ }
+ LIR_Opr thrd = getThreadPointer();
+ LIR_Address* mark_active_flag_addr =
+ new LIR_Address(thrd,
+ in_bytes(JavaThread::satb_mark_queue_offset() +
+ PtrQueue::byte_offset_of_active()),
+ flag_type);
+ // Read the marking-in-progress flag.
+ LIR_Opr flag_val = new_register(T_INT);
+ __ load(mark_active_flag_addr, flag_val);
+
+ LabelObj* start_store = new LabelObj();
+
+ LIR_PatchCode pre_val_patch_code =
+ patch ? lir_patch_normal : lir_patch_none;
+
+ LIR_Opr pre_val = new_register(T_OBJECT);
+
+ __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
+ if (!addr_opr->is_address()) {
+ assert(addr_opr->is_register(), "must be");
+ addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, 0, T_OBJECT));
+ }
+ CodeStub* slow = new G1PreBarrierStub(addr_opr, pre_val, pre_val_patch_code,
+ info);
+ __ branch(lir_cond_notEqual, T_INT, slow);
+ __ branch_destination(slow->continuation());
+}
+
+void LIRGenerator::G1SATBCardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
+ if (G1DisablePostBarrier) return;
+
+ // If the "new_val" is a constant NULL, no barrier is necessary.
+ if (new_val->is_constant() &&
+ new_val->as_constant_ptr()->as_jobject() == NULL) return;
+
+ if (!new_val->is_register()) {
+ LIR_Opr new_val_reg = new_pointer_register();
+ if (new_val->is_constant()) {
+ __ move(new_val, new_val_reg);
+ } else {
+ __ leal(new_val, new_val_reg);
+ }
+ new_val = new_val_reg;
+ }
+ assert(new_val->is_register(), "must be a register at this point");
+
+ if (addr->is_address()) {
+ LIR_Address* address = addr->as_address_ptr();
+ LIR_Opr ptr = new_pointer_register();
+ if (!address->index()->is_valid() && address->disp() == 0) {
+ __ move(address->base(), ptr);
+ } else {
+ assert(address->disp() != max_jint, "lea doesn't support patched addresses!");
+ __ leal(addr, ptr);
+ }
+ addr = ptr;
+ }
+ assert(addr->is_register(), "must be a register at this point");
+
+ LIR_Opr xor_res = new_pointer_register();
+ LIR_Opr xor_shift_res = new_pointer_register();
+
+ if (TwoOperandLIRForm ) {
+ __ move(addr, xor_res);
+ __ logical_xor(xor_res, new_val, xor_res);
+ __ move(xor_res, xor_shift_res);
+ __ unsigned_shift_right(xor_shift_res,
+ LIR_OprFact::intConst(HeapRegion::LogOfHRGrainBytes),
+ xor_shift_res,
+ LIR_OprDesc::illegalOpr());
+ } else {
+ __ logical_xor(addr, new_val, xor_res);
+ __ unsigned_shift_right(xor_res,
+ LIR_OprFact::intConst(HeapRegion::LogOfHRGrainBytes),
+ xor_shift_res,
+ LIR_OprDesc::illegalOpr());
+ }
+
+ if (!new_val->is_register()) {
+ LIR_Opr new_val_reg = new_pointer_register();
+ __ leal(new_val, new_val_reg);
+ new_val = new_val_reg;
+ }
+ assert(new_val->is_register(), "must be a register at this point");
+
+ __ cmp(lir_cond_notEqual, xor_shift_res, LIR_OprFact::intptrConst(NULL_WORD));
+
+ CodeStub* slow = new G1PostBarrierStub(addr, new_val);
+ __ branch(lir_cond_notEqual, T_INT, slow);
+ __ branch_destination(slow->continuation());
+}
+
+#endif // SERIALGC
+////////////////////////////////////////////////////////////////////////
+
void LIRGenerator::CardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val) {
- BarrierSet* bs = Universe::heap()->barrier_set();
- assert(sizeof(*((CardTableModRefBS*)bs)->byte_map_base) == sizeof(jbyte), "adjust this code");
- LIR_Const* card_table_base = new LIR_Const(((CardTableModRefBS*)bs)->byte_map_base);
+ assert(sizeof(*((CardTableModRefBS*)_bs)->byte_map_base) == sizeof(jbyte), "adjust this code");
+ LIR_Const* card_table_base = new LIR_Const(((CardTableModRefBS*)_bs)->byte_map_base);
if (addr->is_address()) {
LIR_Address* address = addr->as_address_ptr();
LIR_Opr ptr = new_register(T_OBJECT);
@@ -1388,6 +1517,13 @@ void LIRGenerator::do_StoreField(StoreField* x) {
__ membar_release();
}
+ if (is_oop) {
+ // Do the pre-write barrier, if any.
+ pre_barrier(LIR_OprFact::address(address),
+ needs_patching,
+ (info ? new CodeEmitInfo(info) : NULL));
+ }
+
if (is_volatile) {
assert(!needs_patching && x->is_loaded(),
"how do we know it's volatile if it's not loaded");
@@ -1398,7 +1534,12 @@ void LIRGenerator::do_StoreField(StoreField* x) {
}
if (is_oop) {
+#ifdef PRECISE_CARDMARK
+ // Precise cardmarks don't work
+ post_barrier(LIR_OprFact::address(address), value.result());
+#else
post_barrier(object.result(), value.result());
+#endif // PRECISE_CARDMARK
}
if (is_volatile && os::is_MP()) {
diff --git a/hotspot/src/share/vm/c1/c1_LIRGenerator.hpp b/hotspot/src/share/vm/c1/c1_LIRGenerator.hpp
index 1b70887883f..1b8555b2055 100644
--- a/hotspot/src/share/vm/c1/c1_LIRGenerator.hpp
+++ b/hotspot/src/share/vm/c1/c1_LIRGenerator.hpp
@@ -145,6 +145,7 @@ class PhiResolver: public CompilationResourceObj {
// only the classes below belong in the same file
class LIRGenerator: public InstructionVisitor, public BlockClosure {
+
private:
Compilation* _compilation;
ciMethod* _method; // method that we are compiling
@@ -154,6 +155,7 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
Values _instruction_for_operand;
BitMap2D _vreg_flags; // flags which can be set on a per-vreg basis
LIR_List* _lir;
+ BarrierSet* _bs;
LIRGenerator* gen() {
return this;
@@ -174,8 +176,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
LIR_OprList _reg_for_constants;
Values _unpinned_constants;
- LIR_Const* _card_table_base;
-
friend class PhiResolver;
// unified bailout support
@@ -196,8 +196,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
LIR_Opr load_constant(Constant* x);
LIR_Opr load_constant(LIR_Const* constant);
- LIR_Const* card_table_base() const { return _card_table_base; }
-
void set_result(Value x, LIR_Opr opr) {
assert(opr->is_valid(), "must set to valid value");
assert(x->operand()->is_illegal(), "operand should never change");
@@ -253,12 +251,17 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
// generic interface
+ void pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info);
void post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
// specific implementations
+ // pre barriers
+
+ void G1SATBCardTableModRef_pre_barrier(LIR_Opr addr_opr, bool patch, CodeEmitInfo* info);
// post barriers
+ void G1SATBCardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
void CardTableModRef_post_barrier(LIR_OprDesc* addr, LIR_OprDesc* new_val);
diff --git a/hotspot/src/share/vm/c1/c1_Optimizer.cpp b/hotspot/src/share/vm/c1/c1_Optimizer.cpp
index ea4b3a576db..1b9f77aeef2 100644
--- a/hotspot/src/share/vm/c1/c1_Optimizer.cpp
+++ b/hotspot/src/share/vm/c1/c1_Optimizer.cpp
@@ -327,8 +327,6 @@ class BlockMerger: public BlockClosure {
BlockBegin* fsux = if_->fsux();
if (swapped) {
cond = Instruction::mirror(cond);
- tsux = if_->fsux();
- fsux = if_->tsux();
}
BlockBegin* tblock = tval->compare(cond, con, tsux, fsux);
diff --git a/hotspot/src/share/vm/c1/c1_Runtime1.cpp b/hotspot/src/share/vm/c1/c1_Runtime1.cpp
index 4a4765099e5..dcd0f282ab9 100644
--- a/hotspot/src/share/vm/c1/c1_Runtime1.cpp
+++ b/hotspot/src/share/vm/c1/c1_Runtime1.cpp
@@ -168,6 +168,8 @@ void Runtime1::generate_blob_for(StubID id) {
switch (id) {
// These stubs don't need to have an oopmap
case dtrace_object_alloc_id:
+ case g1_pre_barrier_slow_id:
+ case g1_post_barrier_slow_id:
case slow_subtype_check_id:
case fpu2long_stub_id:
case unwind_exception_id:
@@ -840,6 +842,13 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* thread, Runtime1::StubID stub_i
if (TracePatching) {
tty->print_cr("Deoptimizing for patching volatile field reference");
}
+ // It's possible the nmethod was invalidated in the last
+ // safepoint, but if it's still alive then make it not_entrant.
+ nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
+ if (nm != NULL) {
+ nm->make_not_entrant();
+ }
+
VM_DeoptimizeFrame deopt(thread, caller_frame.id());
VMThread::execute(&deopt);
diff --git a/hotspot/src/share/vm/c1/c1_Runtime1.hpp b/hotspot/src/share/vm/c1/c1_Runtime1.hpp
index df6c03883f3..4380b72ee89 100644
--- a/hotspot/src/share/vm/c1/c1_Runtime1.hpp
+++ b/hotspot/src/share/vm/c1/c1_Runtime1.hpp
@@ -56,6 +56,8 @@ class StubAssembler;
stub(access_field_patching) \
stub(load_klass_patching) \
stub(jvmti_exception_throw) \
+ stub(g1_pre_barrier_slow) \
+ stub(g1_post_barrier_slow) \
stub(fpu2long_stub) \
stub(counter_overflow) \
last_entry(number_of_ids)
diff --git a/hotspot/src/share/vm/c1/c1_ValueMap.hpp b/hotspot/src/share/vm/c1/c1_ValueMap.hpp
index efc32200910..8e2d0cb2eab 100644
--- a/hotspot/src/share/vm/c1/c1_ValueMap.hpp
+++ b/hotspot/src/share/vm/c1/c1_ValueMap.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1999-2006 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1999-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -133,53 +133,77 @@ class ValueNumberingVisitor: public InstructionVisitor {
virtual void kill_array(ValueType* type) = 0;
// visitor functions
- void do_StoreField (StoreField* x) { kill_field(x->field()); };
- void do_StoreIndexed (StoreIndexed* x) { kill_array(x->type()); };
- void do_MonitorEnter (MonitorEnter* x) { kill_memory(); };
- void do_MonitorExit (MonitorExit* x) { kill_memory(); };
- void do_Invoke (Invoke* x) { kill_memory(); };
- void do_UnsafePutRaw (UnsafePutRaw* x) { kill_memory(); };
- void do_UnsafePutObject(UnsafePutObject* x) { kill_memory(); };
- void do_Intrinsic (Intrinsic* x) { if (!x->preserves_state()) kill_memory(); };
+ void do_StoreField (StoreField* x) {
+ if (!x->is_initialized()) {
+ kill_memory();
+ } else {
+ kill_field(x->field());
+ }
+ }
+ void do_StoreIndexed (StoreIndexed* x) { kill_array(x->type()); }
+ void do_MonitorEnter (MonitorEnter* x) { kill_memory(); }
+ void do_MonitorExit (MonitorExit* x) { kill_memory(); }
+ void do_Invoke (Invoke* x) { kill_memory(); }
+ void do_UnsafePutRaw (UnsafePutRaw* x) { kill_memory(); }
+ void do_UnsafePutObject(UnsafePutObject* x) { kill_memory(); }
+ void do_Intrinsic (Intrinsic* x) { if (!x->preserves_state()) kill_memory(); }
- void do_Phi (Phi* x) { /* nothing to do */ };
- void do_Local (Local* x) { /* nothing to do */ };
- void do_Constant (Constant* x) { /* nothing to do */ };
- void do_LoadField (LoadField* x) { /* nothing to do */ };
- void do_ArrayLength (ArrayLength* x) { /* nothing to do */ };
- void do_LoadIndexed (LoadIndexed* x) { /* nothing to do */ };
- void do_NegateOp (NegateOp* x) { /* nothing to do */ };
- void do_ArithmeticOp (ArithmeticOp* x) { /* nothing to do */ };
- void do_ShiftOp (ShiftOp* x) { /* nothing to do */ };
- void do_LogicOp (LogicOp* x) { /* nothing to do */ };
- void do_CompareOp (CompareOp* x) { /* nothing to do */ };
- void do_IfOp (IfOp* x) { /* nothing to do */ };
- void do_Convert (Convert* x) { /* nothing to do */ };
- void do_NullCheck (NullCheck* x) { /* nothing to do */ };
- void do_NewInstance (NewInstance* x) { /* nothing to do */ };
- void do_NewTypeArray (NewTypeArray* x) { /* nothing to do */ };
- void do_NewObjectArray (NewObjectArray* x) { /* nothing to do */ };
- void do_NewMultiArray (NewMultiArray* x) { /* nothing to do */ };
- void do_CheckCast (CheckCast* x) { /* nothing to do */ };
- void do_InstanceOf (InstanceOf* x) { /* nothing to do */ };
- void do_BlockBegin (BlockBegin* x) { /* nothing to do */ };
- void do_Goto (Goto* x) { /* nothing to do */ };
- void do_If (If* x) { /* nothing to do */ };
- void do_IfInstanceOf (IfInstanceOf* x) { /* nothing to do */ };
- void do_TableSwitch (TableSwitch* x) { /* nothing to do */ };
- void do_LookupSwitch (LookupSwitch* x) { /* nothing to do */ };
- void do_Return (Return* x) { /* nothing to do */ };
- void do_Throw (Throw* x) { /* nothing to do */ };
- void do_Base (Base* x) { /* nothing to do */ };
- void do_OsrEntry (OsrEntry* x) { /* nothing to do */ };
- void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ };
- void do_RoundFP (RoundFP* x) { /* nothing to do */ };
- void do_UnsafeGetRaw (UnsafeGetRaw* x) { /* nothing to do */ };
- void do_UnsafeGetObject(UnsafeGetObject* x) { /* nothing to do */ };
- void do_UnsafePrefetchRead (UnsafePrefetchRead* x) { /* nothing to do */ };
- void do_UnsafePrefetchWrite(UnsafePrefetchWrite* x) { /* nothing to do */ };
- void do_ProfileCall (ProfileCall* x) { /* nothing to do */ };
- void do_ProfileCounter (ProfileCounter* x) { /* nothing to do */ };
+ void do_Phi (Phi* x) { /* nothing to do */ }
+ void do_Local (Local* x) { /* nothing to do */ }
+ void do_Constant (Constant* x) { /* nothing to do */ }
+ void do_LoadField (LoadField* x) {
+ if (!x->is_initialized()) {
+ kill_memory();
+ }
+ }
+ void do_ArrayLength (ArrayLength* x) { /* nothing to do */ }
+ void do_LoadIndexed (LoadIndexed* x) { /* nothing to do */ }
+ void do_NegateOp (NegateOp* x) { /* nothing to do */ }
+ void do_ArithmeticOp (ArithmeticOp* x) { /* nothing to do */ }
+ void do_ShiftOp (ShiftOp* x) { /* nothing to do */ }
+ void do_LogicOp (LogicOp* x) { /* nothing to do */ }
+ void do_CompareOp (CompareOp* x) { /* nothing to do */ }
+ void do_IfOp (IfOp* x) { /* nothing to do */ }
+ void do_Convert (Convert* x) { /* nothing to do */ }
+ void do_NullCheck (NullCheck* x) { /* nothing to do */ }
+ void do_NewInstance (NewInstance* x) { /* nothing to do */ }
+ void do_NewTypeArray (NewTypeArray* x) { /* nothing to do */ }
+ void do_NewObjectArray (NewObjectArray* x) { /* nothing to do */ }
+ void do_NewMultiArray (NewMultiArray* x) { /* nothing to do */ }
+ void do_CheckCast (CheckCast* x) { /* nothing to do */ }
+ void do_InstanceOf (InstanceOf* x) { /* nothing to do */ }
+ void do_BlockBegin (BlockBegin* x) { /* nothing to do */ }
+ void do_Goto (Goto* x) { /* nothing to do */ }
+ void do_If (If* x) { /* nothing to do */ }
+ void do_IfInstanceOf (IfInstanceOf* x) { /* nothing to do */ }
+ void do_TableSwitch (TableSwitch* x) { /* nothing to do */ }
+ void do_LookupSwitch (LookupSwitch* x) { /* nothing to do */ }
+ void do_Return (Return* x) { /* nothing to do */ }
+ void do_Throw (Throw* x) { /* nothing to do */ }
+ void do_Base (Base* x) { /* nothing to do */ }
+ void do_OsrEntry (OsrEntry* x) { /* nothing to do */ }
+ void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ }
+ void do_RoundFP (RoundFP* x) { /* nothing to do */ }
+ void do_UnsafeGetRaw (UnsafeGetRaw* x) { /* nothing to do */ }
+ void do_UnsafeGetObject(UnsafeGetObject* x) { /* nothing to do */ }
+ void do_UnsafePrefetchRead (UnsafePrefetchRead* x) { /* nothing to do */ }
+ void do_UnsafePrefetchWrite(UnsafePrefetchWrite* x) { /* nothing to do */ }
+ void do_ProfileCall (ProfileCall* x) { /* nothing to do */ }
+ void do_ProfileCounter (ProfileCounter* x) { /* nothing to do */ }
+};
+
+
+class ValueNumberingEffects: public ValueNumberingVisitor {
+ private:
+ ValueMap* _map;
+
+ public:
+ // implementation for abstract methods of ValueNumberingVisitor
+ void kill_memory() { _map->kill_memory(); }
+ void kill_field(ciField* field) { _map->kill_field(field); }
+ void kill_array(ValueType* type) { _map->kill_array(type); }
+
+ ValueNumberingEffects(ValueMap* map): _map(map) {}
};
diff --git a/hotspot/src/share/vm/c1/c1_globals.hpp b/hotspot/src/share/vm/c1/c1_globals.hpp
index 52a10b72490..87efd1b04da 100644
--- a/hotspot/src/share/vm/c1/c1_globals.hpp
+++ b/hotspot/src/share/vm/c1/c1_globals.hpp
@@ -213,9 +213,6 @@
develop(bool, UseFastLocking, true, \
"Use fast inlined locking code") \
\
- product(bool, FastTLABRefill, true, \
- "Use fast TLAB refill code") \
- \
develop(bool, UseSlowPath, false, \
"For debugging: test slow cases by always using them") \
\
diff --git a/hotspot/src/share/vm/ci/ciEnv.cpp b/hotspot/src/share/vm/ci/ciEnv.cpp
index 1d279540e11..de75c734ac0 100644
--- a/hotspot/src/share/vm/ci/ciEnv.cpp
+++ b/hotspot/src/share/vm/ci/ciEnv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1999-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1999-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -484,11 +484,16 @@ ciConstant ciEnv::get_constant_by_index_impl(ciInstanceKlass* accessor,
} else if (tag.is_double()) {
return ciConstant((jdouble)cpool->double_at(index));
} else if (tag.is_string() || tag.is_unresolved_string()) {
- oop string = cpool->string_at(index, THREAD);
- if (HAS_PENDING_EXCEPTION) {
- CLEAR_PENDING_EXCEPTION;
- record_out_of_memory_failure();
- return ciConstant();
+ oop string = NULL;
+ if (cpool->is_pseudo_string_at(index)) {
+ string = cpool->pseudo_string_at(index);
+ } else {
+ string = cpool->string_at(index, THREAD);
+ if (HAS_PENDING_EXCEPTION) {
+ CLEAR_PENDING_EXCEPTION;
+ record_out_of_memory_failure();
+ return ciConstant();
+ }
}
ciObject* constant = get_object(string);
assert (constant->is_instance(), "must be an instance, or not? ");
diff --git a/hotspot/src/share/vm/ci/ciMethodBlocks.cpp b/hotspot/src/share/vm/ci/ciMethodBlocks.cpp
index 2810523f4fc..4f7e52559e6 100644
--- a/hotspot/src/share/vm/ci/ciMethodBlocks.cpp
+++ b/hotspot/src/share/vm/ci/ciMethodBlocks.cpp
@@ -49,7 +49,7 @@ bool ciMethodBlocks::is_block_start(int bci) {
// first half. Returns the range beginning at bci.
ciBlock *ciMethodBlocks::split_block_at(int bci) {
ciBlock *former_block = block_containing(bci);
- ciBlock *new_block = new(_arena) ciBlock(_method, _num_blocks++, this, former_block->start_bci());
+ ciBlock *new_block = new(_arena) ciBlock(_method, _num_blocks++, former_block->start_bci());
_blocks->append(new_block);
assert(former_block != NULL, "must not be NULL");
new_block->set_limit_bci(bci);
@@ -83,7 +83,7 @@ ciBlock *ciMethodBlocks::make_block_at(int bci) {
if (cb == NULL ) {
// This is our first time visiting this bytecode. Create
// a fresh block and assign it this starting point.
- ciBlock *nb = new(_arena) ciBlock(_method, _num_blocks++, this, bci);
+ ciBlock *nb = new(_arena) ciBlock(_method, _num_blocks++, bci);
_blocks->append(nb);
_bci_to_block[bci] = nb;
return nb;
@@ -98,6 +98,11 @@ ciBlock *ciMethodBlocks::make_block_at(int bci) {
}
}
+ciBlock *ciMethodBlocks::make_dummy_block() {
+ ciBlock *dum = new(_arena) ciBlock(_method, -1, 0);
+ return dum;
+}
+
void ciMethodBlocks::do_analysis() {
ciBytecodeStream s(_method);
ciBlock *cur_block = block_containing(0);
@@ -253,7 +258,7 @@ ciMethodBlocks::ciMethodBlocks(Arena *arena, ciMethod *meth): _method(meth),
Copy::zero_to_words((HeapWord*) _bci_to_block, b2bsize / sizeof(HeapWord));
// create initial block covering the entire method
- ciBlock *b = new(arena) ciBlock(_method, _num_blocks++, this, 0);
+ ciBlock *b = new(arena) ciBlock(_method, _num_blocks++, 0);
_blocks->append(b);
_bci_to_block[0] = b;
@@ -334,7 +339,7 @@ void ciMethodBlocks::dump() {
#endif
-ciBlock::ciBlock(ciMethod *method, int index, ciMethodBlocks *mb, int start_bci) :
+ciBlock::ciBlock(ciMethod *method, int index, int start_bci) :
#ifndef PRODUCT
_method(method),
#endif
diff --git a/hotspot/src/share/vm/ci/ciMethodBlocks.hpp b/hotspot/src/share/vm/ci/ciMethodBlocks.hpp
index edfdf19b07e..f1a07513d1f 100644
--- a/hotspot/src/share/vm/ci/ciMethodBlocks.hpp
+++ b/hotspot/src/share/vm/ci/ciMethodBlocks.hpp
@@ -48,6 +48,8 @@ public:
int num_blocks() { return _num_blocks;}
void clear_processed();
+ ciBlock *make_dummy_block(); // a block not associated with a bci
+
#ifndef PRODUCT
void dump();
#endif
@@ -81,7 +83,7 @@ public:
fall_through_bci = -1
};
- ciBlock(ciMethod *method, int index, ciMethodBlocks *mb, int start_bci);
+ ciBlock(ciMethod *method, int index, int start_bci);
int start_bci() const { return _start_bci; }
int limit_bci() const { return _limit_bci; }
int control_bci() const { return _control_bci; }
@@ -94,7 +96,6 @@ public:
int ex_limit_bci() const { return _ex_limit_bci; }
bool contains(int bci) const { return start_bci() <= bci && bci < limit_bci(); }
-
// flag handling
bool processed() const { return (_flags & Processed) != 0; }
bool is_handler() const { return (_flags & Handler) != 0; }
diff --git a/hotspot/src/share/vm/ci/ciTypeFlow.cpp b/hotspot/src/share/vm/ci/ciTypeFlow.cpp
index df053f35ea4..08396235067 100644
--- a/hotspot/src/share/vm/ci/ciTypeFlow.cpp
+++ b/hotspot/src/share/vm/ci/ciTypeFlow.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright 2000-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 2000-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -338,8 +338,10 @@ ciTypeFlow::StateVector::StateVector(ciTypeFlow* analyzer) {
}
_trap_bci = -1;
_trap_index = 0;
+ _def_locals.clear();
}
+
// ------------------------------------------------------------------
// ciTypeFlow::get_start_state
//
@@ -735,7 +737,7 @@ void ciTypeFlow::StateVector::do_multianewarray(ciBytecodeStream* str) {
void ciTypeFlow::StateVector::do_new(ciBytecodeStream* str) {
bool will_link;
ciKlass* klass = str->get_klass(will_link);
- if (!will_link) {
+ if (!will_link || str->is_unresolved_klass()) {
trap(str, klass, str->get_klass_index());
} else {
push_object(klass);
@@ -1268,7 +1270,9 @@ bool ciTypeFlow::StateVector::apply_one_bytecode(ciBytecodeStream* str) {
}
case Bytecodes::_iinc:
{
- check_int(local(str->get_index()));
+ int lnum = str->get_index();
+ check_int(local(lnum));
+ store_to_local(lnum);
break;
}
case Bytecodes::_iload: load_local_int(str->get_index()); break;
@@ -1506,6 +1510,46 @@ void ciTypeFlow::StateVector::print_on(outputStream* st) const {
}
#endif
+
+// ------------------------------------------------------------------
+// ciTypeFlow::SuccIter::next
+//
+void ciTypeFlow::SuccIter::next() {
+ int succ_ct = _pred->successors()->length();
+ int next = _index + 1;
+ if (next < succ_ct) {
+ _index = next;
+ _succ = _pred->successors()->at(next);
+ return;
+ }
+ for (int i = next - succ_ct; i < _pred->exceptions()->length(); i++) {
+ // Do not compile any code for unloaded exception types.
+ // Following compiler passes are responsible for doing this also.
+ ciInstanceKlass* exception_klass = _pred->exc_klasses()->at(i);
+ if (exception_klass->is_loaded()) {
+ _index = next;
+ _succ = _pred->exceptions()->at(i);
+ return;
+ }
+ next++;
+ }
+ _index = -1;
+ _succ = NULL;
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::SuccIter::set_succ
+//
+void ciTypeFlow::SuccIter::set_succ(Block* succ) {
+ int succ_ct = _pred->successors()->length();
+ if (_index < succ_ct) {
+ _pred->successors()->at_put(_index, succ);
+ } else {
+ int idx = _index - succ_ct;
+ _pred->exceptions()->at_put(idx, succ);
+ }
+}
+
// ciTypeFlow::Block
//
// A basic block.
@@ -1526,10 +1570,11 @@ ciTypeFlow::Block::Block(ciTypeFlow* outer,
_jsrs = new_jsrs;
_next = NULL;
_on_work_list = false;
- _pre_order = -1; assert(!has_pre_order(), "");
- _private_copy = false;
+ _backedge_copy = false;
+ _exception_entry = false;
_trap_bci = -1;
_trap_index = 0;
+ df_init();
if (CITraceTypeFlow) {
tty->print_cr(">> Created new block");
@@ -1541,55 +1586,13 @@ ciTypeFlow::Block::Block(ciTypeFlow* outer,
}
// ------------------------------------------------------------------
-// ciTypeFlow::Block::clone_loop_head
-//
-ciTypeFlow::Block*
-ciTypeFlow::Block::clone_loop_head(ciTypeFlow* analyzer,
- int branch_bci,
- ciTypeFlow::Block* target,
- ciTypeFlow::JsrSet* jsrs) {
- // Loop optimizations are not performed on Tier1 compiles. Do nothing.
- if (analyzer->env()->comp_level() < CompLevel_full_optimization) {
- return target;
- }
-
- // The current block ends with a branch.
- //
- // If the target block appears to be the test-clause of a for loop, and
- // it is not too large, and it has not yet been cloned, clone it.
- // The pre-existing copy becomes the private clone used only by
- // the initial iteration of the loop. (We know we are simulating
- // the initial iteration right now, since we have never calculated
- // successors before for this block.)
-
- if (branch_bci <= start()
- && (target->limit() - target->start()) <= CICloneLoopTestLimit
- && target->private_copy_count() == 0) {
- // Setting the private_copy bit ensures that the target block cannot be
- // reached by any other paths, such as fall-in from the loop body.
- // The private copy will be accessible only on successor lists
- // created up to this point.
- target->set_private_copy(true);
- if (CITraceTypeFlow) {
- tty->print(">> Cloning a test-clause block ");
- print_value_on(tty);
- tty->cr();
- }
- // If the target is the current block, then later on a new copy of the
- // target block will be created when its bytecodes are reached by
- // an alternate path. (This is the case for loops with the loop
- // head at the bci-wise bottom of the loop, as with pre-1.4.2 javac.)
- //
- // Otherwise, duplicate the target block now and use it immediately.
- // (The case for loops with the loop head at the bci-wise top of the
- // loop, as with 1.4.2 javac.)
- //
- // In either case, the new copy of the block will remain public.
- if (target != this) {
- target = analyzer->block_at(branch_bci, jsrs);
- }
- }
- return target;
+// ciTypeFlow::Block::df_init
+void ciTypeFlow::Block::df_init() {
+ _pre_order = -1; assert(!has_pre_order(), "");
+ _post_order = -1; assert(!has_post_order(), "");
+ _loop = NULL;
+ _irreducible_entry = false;
+ _rpo_next = NULL;
}
// ------------------------------------------------------------------
@@ -1644,7 +1647,6 @@ ciTypeFlow::Block::successors(ciBytecodeStream* str,
case Bytecodes::_ifnull: case Bytecodes::_ifnonnull:
// Our successors are the branch target and the next bci.
branch_bci = str->get_dest();
- clone_loop_head(analyzer, branch_bci, this, jsrs);
_successors =
new (arena) GrowableArray(arena, 2, 0, NULL);
assert(_successors->length() == IF_NOT_TAKEN, "");
@@ -1658,14 +1660,7 @@ ciTypeFlow::Block::successors(ciBytecodeStream* str,
_successors =
new (arena) GrowableArray(arena, 1, 0, NULL);
assert(_successors->length() == GOTO_TARGET, "");
- target = analyzer->block_at(branch_bci, jsrs);
- // If the target block has not been visited yet, and looks like
- // a two-way branch, attempt to clone it if it is a loop head.
- if (target->_successors != NULL
- && target->_successors->length() == (IF_TAKEN + 1)) {
- target = clone_loop_head(analyzer, branch_bci, target, jsrs);
- }
- _successors->append(target);
+ _successors->append(analyzer->block_at(branch_bci, jsrs));
break;
case Bytecodes::_jsr:
@@ -1801,65 +1796,60 @@ void ciTypeFlow::Block::compute_exceptions() {
}
// ------------------------------------------------------------------
-// ciTypeFlow::Block::is_simpler_than
-//
-// A relation used to order our work list. We work on a block earlier
-// if it has a smaller jsr stack or it occurs earlier in the program
-// text.
-//
-// Note: maybe we should redo this functionality to make blocks
-// which correspond to exceptions lower priority.
-bool ciTypeFlow::Block::is_simpler_than(ciTypeFlow::Block* other) {
- if (other == NULL) {
- return true;
- } else {
- int size1 = _jsrs->size();
- int size2 = other->_jsrs->size();
- if (size1 < size2) {
- return true;
- } else if (size2 < size1) {
- return false;
- } else {
-#if 0
- if (size1 > 0) {
- int r1 = _jsrs->record_at(0)->return_address();
- int r2 = _jsrs->record_at(0)->return_address();
- if (r1 < r2) {
- return true;
- } else if (r2 < r1) {
- return false;
- } else {
- int e1 = _jsrs->record_at(0)->return_address();
- int e2 = _jsrs->record_at(0)->return_address();
- if (e1 < e2) {
- return true;
- } else if (e2 < e1) {
- return false;
- }
- }
- }
-#endif
- return (start() <= other->start());
- }
- }
+// ciTypeFlow::Block::set_backedge_copy
+// Use this only to make a pre-existing public block into a backedge copy.
+void ciTypeFlow::Block::set_backedge_copy(bool z) {
+ assert(z || (z == is_backedge_copy()), "cannot make a backedge copy public");
+ _backedge_copy = z;
}
// ------------------------------------------------------------------
-// ciTypeFlow::Block::set_private_copy
-// Use this only to make a pre-existing public block into a private copy.
-void ciTypeFlow::Block::set_private_copy(bool z) {
- assert(z || (z == is_private_copy()), "cannot make a private copy public");
- _private_copy = z;
+// ciTypeFlow::Block::is_clonable_exit
+//
+// At most 2 normal successors, one of which continues looping,
+// and all exceptional successors must exit.
+bool ciTypeFlow::Block::is_clonable_exit(ciTypeFlow::Loop* lp) {
+ int normal_cnt = 0;
+ int in_loop_cnt = 0;
+ for (SuccIter iter(this); !iter.done(); iter.next()) {
+ Block* succ = iter.succ();
+ if (iter.is_normal_ctrl()) {
+ if (++normal_cnt > 2) return false;
+ if (lp->contains(succ->loop())) {
+ if (++in_loop_cnt > 1) return false;
+ }
+ } else {
+ if (lp->contains(succ->loop())) return false;
+ }
+ }
+ return in_loop_cnt == 1;
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::Block::looping_succ
+//
+ciTypeFlow::Block* ciTypeFlow::Block::looping_succ(ciTypeFlow::Loop* lp) {
+ assert(successors()->length() <= 2, "at most 2 normal successors");
+ for (SuccIter iter(this); !iter.done(); iter.next()) {
+ Block* succ = iter.succ();
+ if (lp->contains(succ->loop())) {
+ return succ;
+ }
+ }
+ return NULL;
}
#ifndef PRODUCT
// ------------------------------------------------------------------
// ciTypeFlow::Block::print_value_on
void ciTypeFlow::Block::print_value_on(outputStream* st) const {
- if (has_pre_order()) st->print("#%-2d ", pre_order());
+ if (has_pre_order()) st->print("#%-2d ", pre_order());
+ if (has_rpo()) st->print("rpo#%-2d ", rpo());
st->print("[%d - %d)", start(), limit());
+ if (is_loop_head()) st->print(" lphd");
+ if (is_irreducible_entry()) st->print(" irred");
if (_jsrs->size() > 0) { st->print("/"); _jsrs->print_on(st); }
- if (is_private_copy()) st->print("/private_copy");
+ if (is_backedge_copy()) st->print("/backedge_copy");
}
// ------------------------------------------------------------------
@@ -1871,6 +1861,16 @@ void ciTypeFlow::Block::print_on(outputStream* st) const {
st->print_cr(" ==================================================== ");
st->print (" ");
print_value_on(st);
+ st->print(" Stored locals: "); def_locals()->print_on(st, outer()->method()->max_locals()); tty->cr();
+ if (loop() && loop()->parent() != NULL) {
+ st->print(" loops:");
+ Loop* lp = loop();
+ do {
+ st->print(" %d<-%d", lp->head()->pre_order(),lp->tail()->pre_order());
+ if (lp->is_irreducible()) st->print("(ir)");
+ lp = lp->parent();
+ } while (lp->parent() != NULL);
+ }
st->cr();
_state->print_on(st);
if (_successors == NULL) {
@@ -1907,6 +1907,21 @@ void ciTypeFlow::Block::print_on(outputStream* st) const {
}
#endif
+#ifndef PRODUCT
+// ------------------------------------------------------------------
+// ciTypeFlow::LocalSet::print_on
+void ciTypeFlow::LocalSet::print_on(outputStream* st, int limit) const {
+ st->print("{");
+ for (int i = 0; i < max; i++) {
+ if (test(i)) st->print(" %d", i);
+ }
+ if (limit > max) {
+ st->print(" %d..%d ", max, limit);
+ }
+ st->print(" }");
+}
+#endif
+
// ciTypeFlow
//
// This is a pass over the bytecodes which computes the following:
@@ -1922,12 +1937,11 @@ ciTypeFlow::ciTypeFlow(ciEnv* env, ciMethod* method, int osr_bci) {
_max_locals = method->max_locals();
_max_stack = method->max_stack();
_code_size = method->code_size();
+ _has_irreducible_entry = false;
_osr_bci = osr_bci;
_failure_reason = NULL;
assert(start_bci() >= 0 && start_bci() < code_size() , "correct osr_bci argument");
-
_work_list = NULL;
- _next_pre_order = 0;
_ciblock_count = _methodBlocks->num_blocks();
_idx_to_blocklist = NEW_ARENA_ARRAY(arena(), GrowableArray*, _ciblock_count);
@@ -1949,12 +1963,6 @@ ciTypeFlow::Block* ciTypeFlow::work_list_next() {
_work_list = next_block->next();
next_block->set_next(NULL);
next_block->set_on_work_list(false);
- if (!next_block->has_pre_order()) {
- // Assign "pre_order" as each new block is taken from the work list.
- // This number may be used by following phases to order block visits.
- assert(!have_block_count(), "must not have mapped blocks yet")
- next_block->set_pre_order(_next_pre_order++);
- }
return next_block;
}
@@ -1962,30 +1970,37 @@ ciTypeFlow::Block* ciTypeFlow::work_list_next() {
// ciTypeFlow::add_to_work_list
//
// Add a basic block to our work list.
+// List is sorted by decreasing postorder sort (same as increasing RPO)
void ciTypeFlow::add_to_work_list(ciTypeFlow::Block* block) {
assert(!block->is_on_work_list(), "must not already be on work list");
if (CITraceTypeFlow) {
- tty->print(">> Adding block%s ", block->has_pre_order() ? " (again)" : "");
+ tty->print(">> Adding block ");
block->print_value_on(tty);
tty->print_cr(" to the work list : ");
}
block->set_on_work_list(true);
- if (block->is_simpler_than(_work_list)) {
+
+ // decreasing post order sort
+
+ Block* prev = NULL;
+ Block* current = _work_list;
+ int po = block->post_order();
+ while (current != NULL) {
+ if (!current->has_post_order() || po > current->post_order())
+ break;
+ prev = current;
+ current = current->next();
+ }
+ if (prev == NULL) {
block->set_next(_work_list);
_work_list = block;
} else {
- Block *temp = _work_list;
- while (!block->is_simpler_than(temp->next())) {
- if (CITraceTypeFlow) {
- tty->print(".");
- }
- temp = temp->next();
- }
- block->set_next(temp->next());
- temp->set_next(block);
+ block->set_next(current);
+ prev->set_next(block);
}
+
if (CITraceTypeFlow) {
tty->cr();
}
@@ -2008,7 +2023,7 @@ ciTypeFlow::Block* ciTypeFlow::block_at(int bci, ciTypeFlow::JsrSet* jsrs, Creat
assert(ciblk->start_bci() == bci, "bad ciBlock boundaries");
Block* block = get_block_for(ciblk->index(), jsrs, option);
- assert(block == NULL? (option == no_create): block->is_private_copy() == (option == create_private_copy), "create option consistent with result");
+ assert(block == NULL? (option == no_create): block->is_backedge_copy() == (option == create_backedge_copy), "create option consistent with result");
if (CITraceTypeFlow) {
if (block != NULL) {
@@ -2072,8 +2087,9 @@ void ciTypeFlow::flow_exceptions(GrowableArray* exceptions,
}
if (block->meet_exception(exception_klass, state)) {
- // Block was modified. Add it to the work list.
- if (!block->is_on_work_list()) {
+ // Block was modified and has PO. Add it to the work list.
+ if (block->has_post_order() &&
+ !block->is_on_work_list()) {
add_to_work_list(block);
}
}
@@ -2091,8 +2107,9 @@ void ciTypeFlow::flow_successors(GrowableArray* successors,
for (int i = 0; i < len; i++) {
Block* block = successors->at(i);
if (block->meet(state)) {
- // Block was modified. Add it to the work list.
- if (!block->is_on_work_list()) {
+ // Block was modified and has PO. Add it to the work list.
+ if (block->has_post_order() &&
+ !block->is_on_work_list()) {
add_to_work_list(block);
}
}
@@ -2133,6 +2150,111 @@ bool ciTypeFlow::can_trap(ciBytecodeStream& str) {
return true;
}
+// ------------------------------------------------------------------
+// ciTypeFlow::clone_loop_heads
+//
+// Clone the loop heads
+bool ciTypeFlow::clone_loop_heads(Loop* lp, StateVector* temp_vector, JsrSet* temp_set) {
+ bool rslt = false;
+ for (PreorderLoops iter(loop_tree_root()); !iter.done(); iter.next()) {
+ lp = iter.current();
+ Block* head = lp->head();
+ if (lp == loop_tree_root() ||
+ lp->is_irreducible() ||
+ !head->is_clonable_exit(lp))
+ continue;
+
+ // check not already cloned
+ if (head->backedge_copy_count() != 0)
+ continue;
+
+ // check _no_ shared head below us
+ Loop* ch;
+ for (ch = lp->child(); ch != NULL && ch->head() != head; ch = ch->sibling());
+ if (ch != NULL)
+ continue;
+
+ // Clone head
+ Block* new_head = head->looping_succ(lp);
+ Block* clone = clone_loop_head(lp, temp_vector, temp_set);
+ // Update lp's info
+ clone->set_loop(lp);
+ lp->set_head(new_head);
+ lp->set_tail(clone);
+ // And move original head into outer loop
+ head->set_loop(lp->parent());
+
+ rslt = true;
+ }
+ return rslt;
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::clone_loop_head
+//
+// Clone lp's head and replace tail's successors with clone.
+//
+// |
+// v
+// head <-> body
+// |
+// v
+// exit
+//
+// new_head
+//
+// |
+// v
+// head ----------\
+// | |
+// | v
+// | clone <-> body
+// | |
+// | /--/
+// | |
+// v v
+// exit
+//
+ciTypeFlow::Block* ciTypeFlow::clone_loop_head(Loop* lp, StateVector* temp_vector, JsrSet* temp_set) {
+ Block* head = lp->head();
+ Block* tail = lp->tail();
+ if (CITraceTypeFlow) {
+ tty->print(">> Requesting clone of loop head "); head->print_value_on(tty);
+ tty->print(" for predecessor "); tail->print_value_on(tty);
+ tty->cr();
+ }
+ Block* clone = block_at(head->start(), head->jsrs(), create_backedge_copy);
+ assert(clone->backedge_copy_count() == 1, "one backedge copy for all back edges");
+
+ assert(!clone->has_pre_order(), "just created");
+ clone->set_next_pre_order();
+
+ // Insert clone after (orig) tail in reverse post order
+ clone->set_rpo_next(tail->rpo_next());
+ tail->set_rpo_next(clone);
+
+ // tail->head becomes tail->clone
+ for (SuccIter iter(tail); !iter.done(); iter.next()) {
+ if (iter.succ() == head) {
+ iter.set_succ(clone);
+ break;
+ }
+ }
+ flow_block(tail, temp_vector, temp_set);
+ if (head == tail) {
+ // For self-loops, clone->head becomes clone->clone
+ flow_block(clone, temp_vector, temp_set);
+ for (SuccIter iter(clone); !iter.done(); iter.next()) {
+ if (iter.succ() == head) {
+ iter.set_succ(clone);
+ break;
+ }
+ }
+ }
+ flow_block(clone, temp_vector, temp_set);
+
+ return clone;
+}
// ------------------------------------------------------------------
// ciTypeFlow::flow_block
@@ -2159,11 +2281,14 @@ void ciTypeFlow::flow_block(ciTypeFlow::Block* block,
// Grab the state from the current block.
block->copy_state_into(state);
+ state->def_locals()->clear();
GrowableArray* exceptions = block->exceptions();
GrowableArray* exc_klasses = block->exc_klasses();
bool has_exceptions = exceptions->length() > 0;
+ bool exceptions_used = false;
+
ciBytecodeStream str(method());
str.reset_to_bci(start);
Bytecodes::Code code;
@@ -2172,6 +2297,7 @@ void ciTypeFlow::flow_block(ciTypeFlow::Block* block,
// Check for exceptional control flow from this point.
if (has_exceptions && can_trap(str)) {
flow_exceptions(exceptions, exc_klasses, state);
+ exceptions_used = true;
}
// Apply the effects of the current bytecode to our state.
bool res = state->apply_one_bytecode(&str);
@@ -2189,9 +2315,14 @@ void ciTypeFlow::flow_block(ciTypeFlow::Block* block,
block->print_on(tty);
}
+ // Save set of locals defined in this block
+ block->def_locals()->add(state->def_locals());
+
// Record (no) successors.
block->successors(&str, state, jsrs);
+ assert(!has_exceptions || exceptions_used, "Not removing exceptions");
+
// Discontinue interpretation of this Block.
return;
}
@@ -2202,6 +2333,7 @@ void ciTypeFlow::flow_block(ciTypeFlow::Block* block,
// Check for exceptional control flow from this point.
if (has_exceptions && can_trap(str)) {
flow_exceptions(exceptions, exc_klasses, state);
+ exceptions_used = true;
}
// Fix the JsrSet to reflect effect of the bytecode.
@@ -2218,10 +2350,305 @@ void ciTypeFlow::flow_block(ciTypeFlow::Block* block,
successors = block->successors(&str, NULL, NULL);
}
+ // Save set of locals defined in this block
+ block->def_locals()->add(state->def_locals());
+
+ // Remove untaken exception paths
+ if (!exceptions_used)
+ exceptions->clear();
+
// Pass our state to successors.
flow_successors(successors, state);
}
+// ------------------------------------------------------------------
+// ciTypeFlow::PostOrderLoops::next
+//
+// Advance to next loop tree using a postorder, left-to-right traversal.
+void ciTypeFlow::PostorderLoops::next() {
+ assert(!done(), "must not be done.");
+ if (_current->sibling() != NULL) {
+ _current = _current->sibling();
+ while (_current->child() != NULL) {
+ _current = _current->child();
+ }
+ } else {
+ _current = _current->parent();
+ }
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::PreOrderLoops::next
+//
+// Advance to next loop tree using a preorder, left-to-right traversal.
+void ciTypeFlow::PreorderLoops::next() {
+ assert(!done(), "must not be done.");
+ if (_current->child() != NULL) {
+ _current = _current->child();
+ } else if (_current->sibling() != NULL) {
+ _current = _current->sibling();
+ } else {
+ while (_current != _root && _current->sibling() == NULL) {
+ _current = _current->parent();
+ }
+ if (_current == _root) {
+ _current = NULL;
+ assert(done(), "must be done.");
+ } else {
+ assert(_current->sibling() != NULL, "must be more to do");
+ _current = _current->sibling();
+ }
+ }
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::Loop::sorted_merge
+//
+// Merge the branch lp into this branch, sorting on the loop head
+// pre_orders. Returns the leaf of the merged branch.
+// Child and sibling pointers will be setup later.
+// Sort is (looking from leaf towards the root)
+// descending on primary key: loop head's pre_order, and
+// ascending on secondary key: loop tail's pre_order.
+ciTypeFlow::Loop* ciTypeFlow::Loop::sorted_merge(Loop* lp) {
+ Loop* leaf = this;
+ Loop* prev = NULL;
+ Loop* current = leaf;
+ while (lp != NULL) {
+ int lp_pre_order = lp->head()->pre_order();
+ // Find insertion point for "lp"
+ while (current != NULL) {
+ if (current == lp)
+ return leaf; // Already in list
+ if (current->head()->pre_order() < lp_pre_order)
+ break;
+ if (current->head()->pre_order() == lp_pre_order &&
+ current->tail()->pre_order() > lp->tail()->pre_order()) {
+ break;
+ }
+ prev = current;
+ current = current->parent();
+ }
+ Loop* next_lp = lp->parent(); // Save future list of items to insert
+ // Insert lp before current
+ lp->set_parent(current);
+ if (prev != NULL) {
+ prev->set_parent(lp);
+ } else {
+ leaf = lp;
+ }
+ prev = lp; // Inserted item is new prev[ious]
+ lp = next_lp; // Next item to insert
+ }
+ return leaf;
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::build_loop_tree
+//
+// Incrementally build loop tree.
+void ciTypeFlow::build_loop_tree(Block* blk) {
+ assert(!blk->is_post_visited(), "precondition");
+ Loop* innermost = NULL; // merge of loop tree branches over all successors
+
+ for (SuccIter iter(blk); !iter.done(); iter.next()) {
+ Loop* lp = NULL;
+ Block* succ = iter.succ();
+ if (!succ->is_post_visited()) {
+ // Found backedge since predecessor post visited, but successor is not
+ assert(succ->pre_order() <= blk->pre_order(), "should be backedge");
+
+ // Create a LoopNode to mark this loop.
+ lp = new (arena()) Loop(succ, blk);
+ if (succ->loop() == NULL)
+ succ->set_loop(lp);
+ // succ->loop will be updated to innermost loop on a later call, when blk==succ
+
+ } else { // Nested loop
+ lp = succ->loop();
+
+ // If succ is loop head, find outer loop.
+ while (lp != NULL && lp->head() == succ) {
+ lp = lp->parent();
+ }
+ if (lp == NULL) {
+ // Infinite loop, it's parent is the root
+ lp = loop_tree_root();
+ }
+ }
+
+ // Check for irreducible loop.
+ // Successor has already been visited. If the successor's loop head
+ // has already been post-visited, then this is another entry into the loop.
+ while (lp->head()->is_post_visited() && lp != loop_tree_root()) {
+ _has_irreducible_entry = true;
+ lp->set_irreducible(succ);
+ if (!succ->is_on_work_list()) {
+ // Assume irreducible entries need more data flow
+ add_to_work_list(succ);
+ }
+ lp = lp->parent();
+ assert(lp != NULL, "nested loop must have parent by now");
+ }
+
+ // Merge loop tree branch for all successors.
+ innermost = innermost == NULL ? lp : innermost->sorted_merge(lp);
+
+ } // end loop
+
+ if (innermost == NULL) {
+ assert(blk->successors()->length() == 0, "CFG exit");
+ blk->set_loop(loop_tree_root());
+ } else if (innermost->head() == blk) {
+ // If loop header, complete the tree pointers
+ if (blk->loop() != innermost) {
+#if ASSERT
+ assert(blk->loop()->head() == innermost->head(), "same head");
+ Loop* dl;
+ for (dl = innermost; dl != NULL && dl != blk->loop(); dl = dl->parent());
+ assert(dl == blk->loop(), "blk->loop() already in innermost list");
+#endif
+ blk->set_loop(innermost);
+ }
+ innermost->def_locals()->add(blk->def_locals());
+ Loop* l = innermost;
+ Loop* p = l->parent();
+ while (p && l->head() == blk) {
+ l->set_sibling(p->child()); // Put self on parents 'next child'
+ p->set_child(l); // Make self the first child of parent
+ p->def_locals()->add(l->def_locals());
+ l = p; // Walk up the parent chain
+ p = l->parent();
+ }
+ } else {
+ blk->set_loop(innermost);
+ innermost->def_locals()->add(blk->def_locals());
+ }
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::Loop::contains
+//
+// Returns true if lp is nested loop.
+bool ciTypeFlow::Loop::contains(ciTypeFlow::Loop* lp) const {
+ assert(lp != NULL, "");
+ if (this == lp || head() == lp->head()) return true;
+ int depth1 = depth();
+ int depth2 = lp->depth();
+ if (depth1 > depth2)
+ return false;
+ while (depth1 < depth2) {
+ depth2--;
+ lp = lp->parent();
+ }
+ return this == lp;
+}
+
+// ------------------------------------------------------------------
+// ciTypeFlow::Loop::depth
+//
+// Loop depth
+int ciTypeFlow::Loop::depth() const {
+ int dp = 0;
+ for (Loop* lp = this->parent(); lp != NULL; lp = lp->parent())
+ dp++;
+ return dp;
+}
+
+#ifndef PRODUCT
+// ------------------------------------------------------------------
+// ciTypeFlow::Loop::print
+void ciTypeFlow::Loop::print(outputStream* st, int indent) const {
+ for (int i = 0; i < indent; i++) st->print(" ");
+ st->print("%d<-%d %s",
+ is_root() ? 0 : this->head()->pre_order(),
+ is_root() ? 0 : this->tail()->pre_order(),
+ is_irreducible()?" irr":"");
+ st->print(" defs: ");
+ def_locals()->print_on(st, _head->outer()->method()->max_locals());
+ st->cr();
+ for (Loop* ch = child(); ch != NULL; ch = ch->sibling())
+ ch->print(st, indent+2);
+}
+#endif
+
+// ------------------------------------------------------------------
+// ciTypeFlow::df_flow_types
+//
+// Perform the depth first type flow analysis. Helper for flow_types.
+void ciTypeFlow::df_flow_types(Block* start,
+ bool do_flow,
+ StateVector* temp_vector,
+ JsrSet* temp_set) {
+ int dft_len = 100;
+ GrowableArray stk(arena(), dft_len, 0, NULL);
+
+ ciBlock* dummy = _methodBlocks->make_dummy_block();
+ JsrSet* root_set = new JsrSet(NULL, 0);
+ Block* root_head = new (arena()) Block(this, dummy, root_set);
+ Block* root_tail = new (arena()) Block(this, dummy, root_set);
+ root_head->set_pre_order(0);
+ root_head->set_post_order(0);
+ root_tail->set_pre_order(max_jint);
+ root_tail->set_post_order(max_jint);
+ set_loop_tree_root(new (arena()) Loop(root_head, root_tail));
+
+ stk.push(start);
+
+ _next_pre_order = 0; // initialize pre_order counter
+ _rpo_list = NULL;
+ int next_po = 0; // initialize post_order counter
+
+ // Compute RPO and the control flow graph
+ int size;
+ while ((size = stk.length()) > 0) {
+ Block* blk = stk.top(); // Leave node on stack
+ if (!blk->is_visited()) {
+ // forward arc in graph
+ assert (!blk->has_pre_order(), "");
+ blk->set_next_pre_order();
+
+ if (_next_pre_order >= MaxNodeLimit / 2) {
+ // Too many basic blocks. Bail out.
+ // This can happen when try/finally constructs are nested to depth N,
+ // and there is O(2**N) cloning of jsr bodies. See bug 4697245!
+ // "MaxNodeLimit / 2" is used because probably the parser will
+ // generate at least twice that many nodes and bail out.
+ record_failure("too many basic blocks");
+ return;
+ }
+ if (do_flow) {
+ flow_block(blk, temp_vector, temp_set);
+ if (failing()) return; // Watch for bailouts.
+ }
+ } else if (!blk->is_post_visited()) {
+ // cross or back arc
+ for (SuccIter iter(blk); !iter.done(); iter.next()) {
+ Block* succ = iter.succ();
+ if (!succ->is_visited()) {
+ stk.push(succ);
+ }
+ }
+ if (stk.length() == size) {
+ // There were no additional children, post visit node now
+ stk.pop(); // Remove node from stack
+
+ build_loop_tree(blk);
+ blk->set_post_order(next_po++); // Assign post order
+ prepend_to_rpo_list(blk);
+ assert(blk->is_post_visited(), "");
+
+ if (blk->is_loop_head() && !blk->is_on_work_list()) {
+ // Assume loop heads need more data flow
+ add_to_work_list(blk);
+ }
+ }
+ } else {
+ stk.pop(); // Remove post-visited node from stack
+ }
+ }
+}
+
// ------------------------------------------------------------------
// ciTypeFlow::flow_types
//
@@ -2233,91 +2660,93 @@ void ciTypeFlow::flow_types() {
JsrSet* temp_set = new JsrSet(NULL, 16);
// Create the method entry block.
- Block* block = block_at(start_bci(), temp_set);
- block->set_pre_order(_next_pre_order++);
- assert(block->is_start(), "start block must have order #0");
+ Block* start = block_at(start_bci(), temp_set);
// Load the initial state into it.
const StateVector* start_state = get_start_state();
if (failing()) return;
- block->meet(start_state);
- add_to_work_list(block);
+ start->meet(start_state);
- // Trickle away.
- while (!work_list_empty()) {
- Block* block = work_list_next();
- flow_block(block, temp_vector, temp_set);
+ // Depth first visit
+ df_flow_types(start, true /*do flow*/, temp_vector, temp_set);
+ if (failing()) return;
+ assert(_rpo_list == start, "must be start");
- // NodeCountCutoff is the number of nodes at which the parser
- // will bail out. Probably if we already have lots of BBs,
- // the parser will generate at least twice that many nodes and bail out.
- // Therefore, this is a conservatively large limit at which to
- // bail out in the pre-parse typeflow pass.
- int block_limit = MaxNodeLimit / 2;
+ // Any loops found?
+ if (loop_tree_root()->child() != NULL &&
+ env()->comp_level() >= CompLevel_full_optimization) {
+ // Loop optimizations are not performed on Tier1 compiles.
- if (_next_pre_order >= block_limit) {
- // Too many basic blocks. Bail out.
- //
- // This can happen when try/finally constructs are nested to depth N,
- // and there is O(2**N) cloning of jsr bodies. See bug 4697245!
- record_failure("too many basic blocks");
- return;
+ bool changed = clone_loop_heads(loop_tree_root(), temp_vector, temp_set);
+
+ // If some loop heads were cloned, recompute postorder and loop tree
+ if (changed) {
+ loop_tree_root()->set_child(NULL);
+ for (Block* blk = _rpo_list; blk != NULL;) {
+ Block* next = blk->rpo_next();
+ blk->df_init();
+ blk = next;
+ }
+ df_flow_types(start, false /*no flow*/, temp_vector, temp_set);
}
+ }
- // Watch for bailouts.
- if (failing()) return;
+ if (CITraceTypeFlow) {
+ tty->print_cr("\nLoop tree");
+ loop_tree_root()->print();
+ }
+
+ // Continue flow analysis until fixed point reached
+
+ debug_only(int max_block = _next_pre_order;)
+
+ while (!work_list_empty()) {
+ Block* blk = work_list_next();
+ assert (blk->has_post_order(), "post order assigned above");
+
+ flow_block(blk, temp_vector, temp_set);
+
+ assert (max_block == _next_pre_order, "no new blocks");
+ assert (!failing(), "no more bailouts");
}
}
// ------------------------------------------------------------------
// ciTypeFlow::map_blocks
//
-// Create the block map, which indexes blocks in pre_order.
+// Create the block map, which indexes blocks in reverse post-order.
void ciTypeFlow::map_blocks() {
assert(_block_map == NULL, "single initialization");
- int pre_order_limit = _next_pre_order;
- _block_map = NEW_ARENA_ARRAY(arena(), Block*, pre_order_limit);
- assert(pre_order_limit == block_count(), "");
- int po;
- for (po = 0; po < pre_order_limit; po++) {
- debug_only(_block_map[po] = NULL);
+ int block_ct = _next_pre_order;
+ _block_map = NEW_ARENA_ARRAY(arena(), Block*, block_ct);
+ assert(block_ct == block_count(), "");
+
+ Block* blk = _rpo_list;
+ for (int m = 0; m < block_ct; m++) {
+ int rpo = blk->rpo();
+ assert(rpo == m, "should be sequential");
+ _block_map[rpo] = blk;
+ blk = blk->rpo_next();
}
- ciMethodBlocks *mblks = _methodBlocks;
- ciBlock* current = NULL;
- int limit_bci = code_size();
- for (int bci = 0; bci < limit_bci; bci++) {
- ciBlock* ciblk = mblks->block_containing(bci);
- if (ciblk != NULL && ciblk != current) {
- current = ciblk;
- int curidx = ciblk->index();
- int block_count = (_idx_to_blocklist[curidx] == NULL) ? 0 : _idx_to_blocklist[curidx]->length();
- for (int i = 0; i < block_count; i++) {
- Block* block = _idx_to_blocklist[curidx]->at(i);
- if (!block->has_pre_order()) continue;
- int po = block->pre_order();
- assert(_block_map[po] == NULL, "unique ref to block");
- assert(0 <= po && po < pre_order_limit, "");
- _block_map[po] = block;
- }
- }
- }
- for (po = 0; po < pre_order_limit; po++) {
- assert(_block_map[po] != NULL, "must not drop any blocks");
- Block* block = _block_map[po];
+ assert(blk == NULL, "should be done");
+
+ for (int j = 0; j < block_ct; j++) {
+ assert(_block_map[j] != NULL, "must not drop any blocks");
+ Block* block = _block_map[j];
// Remove dead blocks from successor lists:
for (int e = 0; e <= 1; e++) {
GrowableArray* l = e? block->exceptions(): block->successors();
- for (int i = 0; i < l->length(); i++) {
- Block* s = l->at(i);
- if (!s->has_pre_order()) {
+ for (int k = 0; k < l->length(); k++) {
+ Block* s = l->at(k);
+ if (!s->has_post_order()) {
if (CITraceTypeFlow) {
tty->print("Removing dead %s successor of #%d: ", (e? "exceptional": "normal"), block->pre_order());
s->print_value_on(tty);
tty->cr();
}
l->remove(s);
- --i;
+ --k;
}
}
}
@@ -2329,7 +2758,7 @@ void ciTypeFlow::map_blocks() {
//
// Find a block with this ciBlock which has a compatible JsrSet.
// If no such block exists, create it, unless the option is no_create.
-// If the option is create_private_copy, always create a fresh private copy.
+// If the option is create_backedge_copy, always create a fresh backedge copy.
ciTypeFlow::Block* ciTypeFlow::get_block_for(int ciBlockIndex, ciTypeFlow::JsrSet* jsrs, CreateOption option) {
Arena* a = arena();
GrowableArray* blocks = _idx_to_blocklist[ciBlockIndex];
@@ -2342,11 +2771,11 @@ ciTypeFlow::Block* ciTypeFlow::get_block_for(int ciBlockIndex, ciTypeFlow::JsrSe
_idx_to_blocklist[ciBlockIndex] = blocks;
}
- if (option != create_private_copy) {
+ if (option != create_backedge_copy) {
int len = blocks->length();
for (int i = 0; i < len; i++) {
Block* block = blocks->at(i);
- if (!block->is_private_copy() && block->is_compatible_with(jsrs)) {
+ if (!block->is_backedge_copy() && block->is_compatible_with(jsrs)) {
return block;
}
}
@@ -2357,15 +2786,15 @@ ciTypeFlow::Block* ciTypeFlow::get_block_for(int ciBlockIndex, ciTypeFlow::JsrSe
// We did not find a compatible block. Create one.
Block* new_block = new (a) Block(this, _methodBlocks->block(ciBlockIndex), jsrs);
- if (option == create_private_copy) new_block->set_private_copy(true);
+ if (option == create_backedge_copy) new_block->set_backedge_copy(true);
blocks->append(new_block);
return new_block;
}
// ------------------------------------------------------------------
-// ciTypeFlow::private_copy_count
+// ciTypeFlow::backedge_copy_count
//
-int ciTypeFlow::private_copy_count(int ciBlockIndex, ciTypeFlow::JsrSet* jsrs) const {
+int ciTypeFlow::backedge_copy_count(int ciBlockIndex, ciTypeFlow::JsrSet* jsrs) const {
GrowableArray* blocks = _idx_to_blocklist[ciBlockIndex];
if (blocks == NULL) {
@@ -2376,7 +2805,7 @@ int ciTypeFlow::private_copy_count(int ciBlockIndex, ciTypeFlow::JsrSet* jsrs) c
int len = blocks->length();
for (int i = 0; i < len; i++) {
Block* block = blocks->at(i);
- if (block->is_private_copy() && block->is_compatible_with(jsrs)) {
+ if (block->is_backedge_copy() && block->is_compatible_with(jsrs)) {
count++;
}
}
@@ -2405,10 +2834,12 @@ void ciTypeFlow::do_flow() {
if (failing()) {
return;
}
- if (CIPrintTypeFlow || CITraceTypeFlow) {
- print_on(tty);
- }
+
map_blocks();
+
+ if (CIPrintTypeFlow || CITraceTypeFlow) {
+ rpo_print_on(tty);
+ }
}
// ------------------------------------------------------------------
@@ -2466,4 +2897,19 @@ void ciTypeFlow::print_on(outputStream* st) const {
st->print_cr("********************************************************");
st->cr();
}
+
+void ciTypeFlow::rpo_print_on(outputStream* st) const {
+ st->print_cr("********************************************************");
+ st->print ("TypeFlow for ");
+ method()->name()->print_symbol_on(st);
+ int limit_bci = code_size();
+ st->print_cr(" %d bytes", limit_bci);
+ for (Block* blk = _rpo_list; blk != NULL; blk = blk->rpo_next()) {
+ blk->print_on(st);
+ st->print_cr("--------------------------------------------------------");
+ st->cr();
+ }
+ st->print_cr("********************************************************");
+ st->cr();
+}
#endif
diff --git a/hotspot/src/share/vm/ci/ciTypeFlow.hpp b/hotspot/src/share/vm/ci/ciTypeFlow.hpp
index f095d662b99..4dae26ea512 100644
--- a/hotspot/src/share/vm/ci/ciTypeFlow.hpp
+++ b/hotspot/src/share/vm/ci/ciTypeFlow.hpp
@@ -34,11 +34,13 @@ private:
int _max_locals;
int _max_stack;
int _code_size;
+ bool _has_irreducible_entry;
const char* _failure_reason;
public:
class StateVector;
+ class Loop;
class Block;
// Build a type flow analyzer
@@ -55,6 +57,7 @@ public:
int max_stack() const { return _max_stack; }
int max_cells() const { return _max_locals + _max_stack; }
int code_size() const { return _code_size; }
+ bool has_irreducible_entry() const { return _has_irreducible_entry; }
// Represents information about an "active" jsr call. This
// class represents a call to the routine at some entry address
@@ -125,6 +128,19 @@ public:
void print_on(outputStream* st) const PRODUCT_RETURN;
};
+ class LocalSet VALUE_OBJ_CLASS_SPEC {
+ private:
+ enum Constants { max = 63 };
+ uint64_t _bits;
+ public:
+ LocalSet() : _bits(0) {}
+ void add(uint32_t i) { if (i < (uint32_t)max) _bits |= (1LL << i); }
+ void add(LocalSet* ls) { _bits |= ls->_bits; }
+ bool test(uint32_t i) const { return i < (uint32_t)max ? (_bits>>i)&1U : true; }
+ void clear() { _bits = 0; }
+ void print_on(outputStream* st, int limit) const PRODUCT_RETURN;
+ };
+
// Used as a combined index for locals and temps
enum Cell {
Cell_0, Cell_max = INT_MAX
@@ -142,6 +158,8 @@ public:
int _trap_bci;
int _trap_index;
+ LocalSet _def_locals; // For entire block
+
static ciType* type_meet_internal(ciType* t1, ciType* t2, ciTypeFlow* analyzer);
public:
@@ -181,6 +199,9 @@ public:
int monitor_count() const { return _monitor_count; }
void set_monitor_count(int mc) { _monitor_count = mc; }
+ LocalSet* def_locals() { return &_def_locals; }
+ const LocalSet* def_locals() const { return &_def_locals; }
+
static Cell start_cell() { return (Cell)0; }
static Cell next_cell(Cell c) { return (Cell)(((int)c) + 1); }
Cell limit_cell() const {
@@ -250,6 +271,10 @@ public:
return type->basic_type() == T_DOUBLE;
}
+ void store_to_local(int lnum) {
+ _def_locals.add((uint) lnum);
+ }
+
void push_translate(ciType* type);
void push_int() {
@@ -358,6 +383,7 @@ public:
"must be reference type or return address");
overwrite_local_double_long(index);
set_type_at(local(index), type);
+ store_to_local(index);
}
void load_local_double(int index) {
@@ -376,6 +402,8 @@ public:
overwrite_local_double_long(index);
set_type_at(local(index), type);
set_type_at(local(index+1), type2);
+ store_to_local(index);
+ store_to_local(index+1);
}
void load_local_float(int index) {
@@ -388,6 +416,7 @@ public:
assert(is_float(type), "must be float type");
overwrite_local_double_long(index);
set_type_at(local(index), type);
+ store_to_local(index);
}
void load_local_int(int index) {
@@ -400,6 +429,7 @@ public:
assert(is_int(type), "must be int type");
overwrite_local_double_long(index);
set_type_at(local(index), type);
+ store_to_local(index);
}
void load_local_long(int index) {
@@ -418,6 +448,8 @@ public:
overwrite_local_double_long(index);
set_type_at(local(index), type);
set_type_at(local(index+1), type2);
+ store_to_local(index);
+ store_to_local(index+1);
}
// Stop interpretation of this path with a trap.
@@ -450,13 +482,31 @@ public:
};
// Parameter for "find_block" calls:
- // Describes the difference between a public and private copy.
+ // Describes the difference between a public and backedge copy.
enum CreateOption {
create_public_copy,
- create_private_copy,
+ create_backedge_copy,
no_create
};
+ // Successor iterator
+ class SuccIter : public StackObj {
+ private:
+ Block* _pred;
+ int _index;
+ Block* _succ;
+ public:
+ SuccIter() : _pred(NULL), _index(-1), _succ(NULL) {}
+ SuccIter(Block* pred) : _pred(pred), _index(-1), _succ(NULL) { next(); }
+ int index() { return _index; }
+ Block* pred() { return _pred; } // Return predecessor
+ bool done() { return _index < 0; } // Finished?
+ Block* succ() { return _succ; } // Return current successor
+ void next(); // Advance
+ void set_succ(Block* succ); // Update current successor
+ bool is_normal_ctrl() { return index() < _pred->successors()->length(); }
+ };
+
// A basic block
class Block : public ResourceObj {
private:
@@ -470,15 +520,24 @@ public:
int _trap_bci;
int _trap_index;
- // A reasonable approximation to pre-order, provided.to the client.
+ // pre_order, assigned at first visit. Used as block ID and "visited" tag
int _pre_order;
- // Has this block been cloned for some special purpose?
- bool _private_copy;
+ // A post-order, used to compute the reverse post order (RPO) provided to the client
+ int _post_order; // used to compute rpo
+
+ // Has this block been cloned for a loop backedge?
+ bool _backedge_copy;
// A pointer used for our internal work list
- Block* _next;
- bool _on_work_list;
+ Block* _next;
+ bool _on_work_list; // on the work list
+ Block* _rpo_next; // Reverse post order list
+
+ // Loop info
+ Loop* _loop; // nearest loop
+ bool _irreducible_entry; // entry to irreducible loop
+ bool _exception_entry; // entry to exception handler
ciBlock* ciblock() const { return _ciblock; }
StateVector* state() const { return _state; }
@@ -504,10 +563,11 @@ public:
int start() const { return _ciblock->start_bci(); }
int limit() const { return _ciblock->limit_bci(); }
int control() const { return _ciblock->control_bci(); }
+ JsrSet* jsrs() const { return _jsrs; }
- bool is_private_copy() const { return _private_copy; }
- void set_private_copy(bool z);
- int private_copy_count() const { return outer()->private_copy_count(ciblock()->index(), _jsrs); }
+ bool is_backedge_copy() const { return _backedge_copy; }
+ void set_backedge_copy(bool z);
+ int backedge_copy_count() const { return outer()->backedge_copy_count(ciblock()->index(), _jsrs); }
// access to entry state
int stack_size() const { return _state->stack_size(); }
@@ -515,6 +575,20 @@ public:
ciType* local_type_at(int i) const { return _state->local_type_at(i); }
ciType* stack_type_at(int i) const { return _state->stack_type_at(i); }
+ // Data flow on locals
+ bool is_invariant_local(uint v) const {
+ assert(is_loop_head(), "only loop heads");
+ // Find outermost loop with same loop head
+ Loop* lp = loop();
+ while (lp->parent() != NULL) {
+ if (lp->parent()->head() != lp->head()) break;
+ lp = lp->parent();
+ }
+ return !lp->def_locals()->test(v);
+ }
+ LocalSet* def_locals() { return _state->def_locals(); }
+ const LocalSet* def_locals() const { return _state->def_locals(); }
+
// Get the successors for this Block.
GrowableArray* successors(ciBytecodeStream* str,
StateVector* state,
@@ -524,13 +598,6 @@ public:
return _successors;
}
- // Helper function for "successors" when making private copies of
- // loop heads for C2.
- Block * clone_loop_head(ciTypeFlow* analyzer,
- int branch_bci,
- Block* target,
- JsrSet* jsrs);
-
// Get the exceptional successors for this Block.
GrowableArray* exceptions() {
if (_exceptions == NULL) {
@@ -584,17 +651,126 @@ public:
bool is_on_work_list() const { return _on_work_list; }
bool has_pre_order() const { return _pre_order >= 0; }
- void set_pre_order(int po) { assert(!has_pre_order() && po >= 0, ""); _pre_order = po; }
+ void set_pre_order(int po) { assert(!has_pre_order(), ""); _pre_order = po; }
int pre_order() const { assert(has_pre_order(), ""); return _pre_order; }
+ void set_next_pre_order() { set_pre_order(outer()->inc_next_pre_order()); }
bool is_start() const { return _pre_order == outer()->start_block_num(); }
- // A ranking used in determining order within the work list.
- bool is_simpler_than(Block* other);
+ // Reverse post order
+ void df_init();
+ bool has_post_order() const { return _post_order >= 0; }
+ void set_post_order(int po) { assert(!has_post_order() && po >= 0, ""); _post_order = po; }
+ void reset_post_order(int o){ _post_order = o; }
+ int post_order() const { assert(has_post_order(), ""); return _post_order; }
+
+ bool has_rpo() const { return has_post_order() && outer()->have_block_count(); }
+ int rpo() const { assert(has_rpo(), ""); return outer()->block_count() - post_order() - 1; }
+ void set_rpo_next(Block* b) { _rpo_next = b; }
+ Block* rpo_next() { return _rpo_next; }
+
+ // Loops
+ Loop* loop() const { return _loop; }
+ void set_loop(Loop* lp) { _loop = lp; }
+ bool is_loop_head() const { return _loop && _loop->head() == this; }
+ void set_irreducible_entry(bool c) { _irreducible_entry = c; }
+ bool is_irreducible_entry() const { return _irreducible_entry; }
+ bool is_visited() const { return has_pre_order(); }
+ bool is_post_visited() const { return has_post_order(); }
+ bool is_clonable_exit(Loop* lp);
+ Block* looping_succ(Loop* lp); // Successor inside of loop
+ bool is_single_entry_loop_head() const {
+ if (!is_loop_head()) return false;
+ for (Loop* lp = loop(); lp != NULL && lp->head() == this; lp = lp->parent())
+ if (lp->is_irreducible()) return false;
+ return true;
+ }
void print_value_on(outputStream* st) const PRODUCT_RETURN;
void print_on(outputStream* st) const PRODUCT_RETURN;
};
+ // Loop
+ class Loop : public ResourceObj {
+ private:
+ Loop* _parent;
+ Loop* _sibling; // List of siblings, null terminated
+ Loop* _child; // Head of child list threaded thru sibling pointer
+ Block* _head; // Head of loop
+ Block* _tail; // Tail of loop
+ bool _irreducible;
+ LocalSet _def_locals;
+
+ public:
+ Loop(Block* head, Block* tail) :
+ _head(head), _tail(tail),
+ _parent(NULL), _sibling(NULL), _child(NULL),
+ _irreducible(false), _def_locals() {}
+
+ Loop* parent() const { return _parent; }
+ Loop* sibling() const { return _sibling; }
+ Loop* child() const { return _child; }
+ Block* head() const { return _head; }
+ Block* tail() const { return _tail; }
+ void set_parent(Loop* p) { _parent = p; }
+ void set_sibling(Loop* s) { _sibling = s; }
+ void set_child(Loop* c) { _child = c; }
+ void set_head(Block* hd) { _head = hd; }
+ void set_tail(Block* tl) { _tail = tl; }
+
+ int depth() const; // nesting depth
+
+ // Returns true if lp is a nested loop or us.
+ bool contains(Loop* lp) const;
+ bool contains(Block* blk) const { return contains(blk->loop()); }
+
+ // Data flow on locals
+ LocalSet* def_locals() { return &_def_locals; }
+ const LocalSet* def_locals() const { return &_def_locals; }
+
+ // Merge the branch lp into this branch, sorting on the loop head
+ // pre_orders. Returns the new branch.
+ Loop* sorted_merge(Loop* lp);
+
+ // Mark non-single entry to loop
+ void set_irreducible(Block* entry) {
+ _irreducible = true;
+ entry->set_irreducible_entry(true);
+ }
+ bool is_irreducible() const { return _irreducible; }
+
+ bool is_root() const { return _tail->pre_order() == max_jint; }
+
+ void print(outputStream* st = tty, int indent = 0) const PRODUCT_RETURN;
+ };
+
+ // Postorder iteration over the loop tree.
+ class PostorderLoops : public StackObj {
+ private:
+ Loop* _root;
+ Loop* _current;
+ public:
+ PostorderLoops(Loop* root) : _root(root), _current(root) {
+ while (_current->child() != NULL) {
+ _current = _current->child();
+ }
+ }
+ bool done() { return _current == NULL; } // Finished iterating?
+ void next(); // Advance to next loop
+ Loop* current() { return _current; } // Return current loop.
+ };
+
+ // Preorder iteration over the loop tree.
+ class PreorderLoops : public StackObj {
+ private:
+ Loop* _root;
+ Loop* _current;
+ public:
+ PreorderLoops(Loop* root) : _root(root), _current(root) {}
+ bool done() { return _current == NULL; } // Finished iterating?
+ void next(); // Advance to next loop
+ Loop* current() { return _current; } // Return current loop.
+ };
+
// Standard indexes of successors, for various bytecodes.
enum {
FALL_THROUGH = 0, // normal control
@@ -619,6 +795,12 @@ private:
// Tells if a given instruction is able to generate an exception edge.
bool can_trap(ciBytecodeStream& str);
+ // Clone the loop heads. Returns true if any cloning occurred.
+ bool clone_loop_heads(Loop* lp, StateVector* temp_vector, JsrSet* temp_set);
+
+ // Clone lp's head and replace tail's successors with clone.
+ Block* clone_loop_head(Loop* lp, StateVector* temp_vector, JsrSet* temp_set);
+
public:
// Return the block beginning at bci which has a JsrSet compatible
// with jsrs.
@@ -627,8 +809,8 @@ public:
// block factory
Block* get_block_for(int ciBlockIndex, JsrSet* jsrs, CreateOption option = create_public_copy);
- // How many of the blocks have the private_copy bit set?
- int private_copy_count(int ciBlockIndex, JsrSet* jsrs) const;
+ // How many of the blocks have the backedge_copy bit set?
+ int backedge_copy_count(int ciBlockIndex, JsrSet* jsrs) const;
// Return an existing block containing bci which has a JsrSet compatible
// with jsrs, or NULL if there is none.
@@ -651,11 +833,18 @@ public:
return _block_map[po]; }
Block* start_block() const { return pre_order_at(start_block_num()); }
int start_block_num() const { return 0; }
+ Block* rpo_at(int rpo) const { assert(0 <= rpo && rpo < block_count(), "out of bounds");
+ return _block_map[rpo]; }
+ int next_pre_order() { return _next_pre_order; }
+ int inc_next_pre_order() { return _next_pre_order++; }
private:
// A work list used during flow analysis.
Block* _work_list;
+ // List of blocks in reverse post order
+ Block* _rpo_list;
+
// Next Block::_pre_order. After mapping, doubles as block_count.
int _next_pre_order;
@@ -668,6 +857,15 @@ private:
// Add a basic block to our work list.
void add_to_work_list(Block* block);
+ // Prepend a basic block to rpo list.
+ void prepend_to_rpo_list(Block* blk) {
+ blk->set_rpo_next(_rpo_list);
+ _rpo_list = blk;
+ }
+
+ // Root of the loop tree
+ Loop* _loop_tree_root;
+
// State used for make_jsr_record
int _jsr_count;
GrowableArray* _jsr_records;
@@ -677,6 +875,9 @@ public:
// does not already exist.
JsrRecord* make_jsr_record(int entry_address, int return_address);
+ void set_loop_tree_root(Loop* ltr) { _loop_tree_root = ltr; }
+ Loop* loop_tree_root() { return _loop_tree_root; }
+
private:
// Get the initial state for start_bci:
const StateVector* get_start_state();
@@ -703,6 +904,15 @@ private:
// necessary.
void flow_types();
+ // Perform the depth first type flow analysis. Helper for flow_types.
+ void df_flow_types(Block* start,
+ bool do_flow,
+ StateVector* temp_vector,
+ JsrSet* temp_set);
+
+ // Incrementally build loop tree.
+ void build_loop_tree(Block* blk);
+
// Create the block map, which indexes blocks in pre_order.
void map_blocks();
@@ -711,4 +921,6 @@ public:
void do_flow();
void print_on(outputStream* st) const PRODUCT_RETURN;
+
+ void rpo_print_on(outputStream* st) const PRODUCT_RETURN;
};
diff --git a/hotspot/src/share/vm/classfile/classFileParser.cpp b/hotspot/src/share/vm/classfile/classFileParser.cpp
index 7ee8ce98403..703217f9255 100644
--- a/hotspot/src/share/vm/classfile/classFileParser.cpp
+++ b/hotspot/src/share/vm/classfile/classFileParser.cpp
@@ -168,11 +168,23 @@ void ClassFileParser::parse_constant_pool_entries(constantPoolHandle cp, int len
// Got utf8 string, guarantee utf8_length+1 bytes, set stream position forward.
cfs->guarantee_more(utf8_length+1, CHECK); // utf8 string, tag/access_flags
cfs->skip_u1_fast(utf8_length);
+
// Before storing the symbol, make sure it's legal
if (_need_verify) {
verify_legal_utf8((unsigned char*)utf8_buffer, utf8_length, CHECK);
}
+ if (AnonymousClasses && has_cp_patch_at(index)) {
+ Handle patch = clear_cp_patch_at(index);
+ guarantee_property(java_lang_String::is_instance(patch()),
+ "Illegal utf8 patch at %d in class file %s",
+ index, CHECK);
+ char* str = java_lang_String::as_utf8_string(patch());
+ // (could use java_lang_String::as_symbol instead, but might as well batch them)
+ utf8_buffer = (u1*) str;
+ utf8_length = (int) strlen(str);
+ }
+
unsigned int hash;
symbolOop result = SymbolTable::lookup_only((char*)utf8_buffer, utf8_length, hash);
if (result == NULL) {
@@ -245,7 +257,7 @@ constantPoolHandle ClassFileParser::parse_constant_pool(TRAPS) {
int klass_ref_index = cp->klass_ref_index_at(index);
int name_and_type_ref_index = cp->name_and_type_ref_index_at(index);
check_property(valid_cp_range(klass_ref_index, length) &&
- cp->tag_at(klass_ref_index).is_klass_reference(),
+ is_klass_reference(cp, klass_ref_index),
"Invalid constant pool index %u in class file %s",
klass_ref_index,
CHECK_(nullHandle));
@@ -326,16 +338,46 @@ constantPoolHandle ClassFileParser::parse_constant_pool(TRAPS) {
} // end of switch
} // end of for
+ if (_cp_patches != NULL) {
+ // need to treat this_class specially...
+ assert(AnonymousClasses, "");
+ int this_class_index;
+ {
+ cfs->guarantee_more(8, CHECK_(nullHandle)); // flags, this_class, super_class, infs_len
+ u1* mark = cfs->current();
+ u2 flags = cfs->get_u2_fast();
+ this_class_index = cfs->get_u2_fast();
+ cfs->set_current(mark); // revert to mark
+ }
+
+ for (index = 1; index < length; index++) { // Index 0 is unused
+ if (has_cp_patch_at(index)) {
+ guarantee_property(index != this_class_index,
+ "Illegal constant pool patch to self at %d in class file %s",
+ index, CHECK_(nullHandle));
+ patch_constant_pool(cp, index, cp_patch_at(index), CHECK_(nullHandle));
+ }
+ }
+ // Ensure that all the patches have been used.
+ for (index = 0; index < _cp_patches->length(); index++) {
+ guarantee_property(!has_cp_patch_at(index),
+ "Unused constant pool patch at %d in class file %s",
+ index, CHECK_(nullHandle));
+ }
+ }
+
if (!_need_verify) {
return cp;
}
// second verification pass - checks the strings are of the right format.
+ // but not yet to the other entries
for (index = 1; index < length; index++) {
jbyte tag = cp->tag_at(index).value();
switch (tag) {
case JVM_CONSTANT_UnresolvedClass: {
symbolHandle class_name(THREAD, cp->unresolved_klass_at(index));
+ // check the name, even if _cp_patches will overwrite it
verify_legal_class_name(class_name, CHECK_(nullHandle));
break;
}
@@ -378,6 +420,73 @@ constantPoolHandle ClassFileParser::parse_constant_pool(TRAPS) {
}
+void ClassFileParser::patch_constant_pool(constantPoolHandle cp, int index, Handle patch, TRAPS) {
+ assert(AnonymousClasses, "");
+ BasicType patch_type = T_VOID;
+ switch (cp->tag_at(index).value()) {
+
+ case JVM_CONSTANT_UnresolvedClass :
+ // Patching a class means pre-resolving it.
+ // The name in the constant pool is ignored.
+ if (patch->klass() == SystemDictionary::class_klass()) { // %%% java_lang_Class::is_instance
+ guarantee_property(!java_lang_Class::is_primitive(patch()),
+ "Illegal class patch at %d in class file %s",
+ index, CHECK);
+ cp->klass_at_put(index, java_lang_Class::as_klassOop(patch()));
+ } else {
+ guarantee_property(java_lang_String::is_instance(patch()),
+ "Illegal class patch at %d in class file %s",
+ index, CHECK);
+ symbolHandle name = java_lang_String::as_symbol(patch(), CHECK);
+ cp->unresolved_klass_at_put(index, name());
+ }
+ break;
+
+ case JVM_CONSTANT_UnresolvedString :
+ // Patching a string means pre-resolving it.
+ // The spelling in the constant pool is ignored.
+ // The constant reference may be any object whatever.
+ // If it is not a real interned string, the constant is referred
+ // to as a "pseudo-string", and must be presented to the CP
+ // explicitly, because it may require scavenging.
+ cp->pseudo_string_at_put(index, patch());
+ break;
+
+ case JVM_CONSTANT_Integer : patch_type = T_INT; goto patch_prim;
+ case JVM_CONSTANT_Float : patch_type = T_FLOAT; goto patch_prim;
+ case JVM_CONSTANT_Long : patch_type = T_LONG; goto patch_prim;
+ case JVM_CONSTANT_Double : patch_type = T_DOUBLE; goto patch_prim;
+ patch_prim:
+ {
+ jvalue value;
+ BasicType value_type = java_lang_boxing_object::get_value(patch(), &value);
+ guarantee_property(value_type == patch_type,
+ "Illegal primitive patch at %d in class file %s",
+ index, CHECK);
+ switch (value_type) {
+ case T_INT: cp->int_at_put(index, value.i); break;
+ case T_FLOAT: cp->float_at_put(index, value.f); break;
+ case T_LONG: cp->long_at_put(index, value.j); break;
+ case T_DOUBLE: cp->double_at_put(index, value.d); break;
+ default: assert(false, "");
+ }
+ }
+ break;
+
+ default:
+ // %%% TODO: put method handles into CONSTANT_InterfaceMethodref, etc.
+ guarantee_property(!has_cp_patch_at(index),
+ "Illegal unexpected patch at %d in class file %s",
+ index, CHECK);
+ return;
+ }
+
+ // On fall-through, mark the patch as used.
+ clear_cp_patch_at(index);
+}
+
+
+
class NameSigHash: public ResourceObj {
public:
symbolOop _name; // name
@@ -448,25 +557,33 @@ objArrayHandle ClassFileParser::parse_interfaces(constantPoolHandle cp,
int index;
for (index = 0; index < length; index++) {
u2 interface_index = cfs->get_u2(CHECK_(nullHandle));
+ KlassHandle interf;
check_property(
valid_cp_range(interface_index, cp->length()) &&
- cp->tag_at(interface_index).is_unresolved_klass(),
+ is_klass_reference(cp, interface_index),
"Interface name has bad constant pool index %u in class file %s",
interface_index, CHECK_(nullHandle));
- symbolHandle unresolved_klass (THREAD, cp->klass_name_at(interface_index));
+ if (cp->tag_at(interface_index).is_klass()) {
+ interf = KlassHandle(THREAD, cp->resolved_klass_at(interface_index));
+ } else {
+ symbolHandle unresolved_klass (THREAD, cp->klass_name_at(interface_index));
- // Don't need to check legal name because it's checked when parsing constant pool.
- // But need to make sure it's not an array type.
- guarantee_property(unresolved_klass->byte_at(0) != JVM_SIGNATURE_ARRAY,
- "Bad interface name in class file %s", CHECK_(nullHandle));
+ // Don't need to check legal name because it's checked when parsing constant pool.
+ // But need to make sure it's not an array type.
+ guarantee_property(unresolved_klass->byte_at(0) != JVM_SIGNATURE_ARRAY,
+ "Bad interface name in class file %s", CHECK_(nullHandle));
- vmtimer->suspend(); // do not count recursive loading twice
- // Call resolve_super so classcircularity is checked
- klassOop k = SystemDictionary::resolve_super_or_fail(class_name,
- unresolved_klass, class_loader, protection_domain,
- false, CHECK_(nullHandle));
- KlassHandle interf (THREAD, k);
- vmtimer->resume();
+ vmtimer->suspend(); // do not count recursive loading twice
+ // Call resolve_super so classcircularity is checked
+ klassOop k = SystemDictionary::resolve_super_or_fail(class_name,
+ unresolved_klass, class_loader, protection_domain,
+ false, CHECK_(nullHandle));
+ interf = KlassHandle(THREAD, k);
+ vmtimer->resume();
+
+ if (LinkWellKnownClasses) // my super type is well known to me
+ cp->klass_at_put(interface_index, interf()); // eagerly resolve
+ }
if (!Klass::cast(interf())->is_interface()) {
THROW_MSG_(vmSymbols::java_lang_IncompatibleClassChangeError(), "Implementing class", nullHandle);
@@ -877,8 +994,7 @@ typeArrayHandle ClassFileParser::parse_exception_table(u4 code_length,
"Illegal exception table handler in class file %s", CHECK_(nullHandle));
if (catch_type_index != 0) {
guarantee_property(valid_cp_range(catch_type_index, cp->length()) &&
- (cp->tag_at(catch_type_index).is_klass() ||
- cp->tag_at(catch_type_index).is_unresolved_klass()),
+ is_klass_reference(cp, catch_type_index),
"Catch type in exception table has bad constant type in class file %s", CHECK_(nullHandle));
}
}
@@ -1117,7 +1233,7 @@ void ClassFileParser::parse_type_array(u2 array_length, u4 code_length, u4* u1_i
} else if (tag == ITEM_Object) {
u2 class_index = u2_array[i2++] = cfs->get_u2(CHECK);
guarantee_property(valid_cp_range(class_index, cp->length()) &&
- cp->tag_at(class_index).is_unresolved_klass(),
+ is_klass_reference(cp, class_index),
"Bad class index %u in StackMap in class file %s",
class_index, CHECK);
} else if (tag == ITEM_Uninitialized) {
@@ -1183,7 +1299,7 @@ u2* ClassFileParser::parse_checked_exceptions(u2* checked_exceptions_length,
checked_exception = cfs->get_u2_fast();
check_property(
valid_cp_range(checked_exception, cp->length()) &&
- cp->tag_at(checked_exception).is_klass_reference(),
+ is_klass_reference(cp, checked_exception),
"Exception name has bad type at constant pool %u in class file %s",
checked_exception, CHECK_NULL);
}
@@ -1918,7 +2034,7 @@ u2 ClassFileParser::parse_classfile_inner_classes_attribute(constantPoolHandle c
check_property(
inner_class_info_index == 0 ||
(valid_cp_range(inner_class_info_index, cp_size) &&
- cp->tag_at(inner_class_info_index).is_klass_reference()),
+ is_klass_reference(cp, inner_class_info_index)),
"inner_class_info_index %u has bad constant type in class file %s",
inner_class_info_index, CHECK_0);
// Outer class index
@@ -1926,7 +2042,7 @@ u2 ClassFileParser::parse_classfile_inner_classes_attribute(constantPoolHandle c
check_property(
outer_class_info_index == 0 ||
(valid_cp_range(outer_class_info_index, cp_size) &&
- cp->tag_at(outer_class_info_index).is_klass_reference()),
+ is_klass_reference(cp, outer_class_info_index)),
"outer_class_info_index %u has bad constant type in class file %s",
outer_class_info_index, CHECK_0);
// Inner class name
@@ -2088,7 +2204,7 @@ void ClassFileParser::parse_classfile_attributes(constantPoolHandle cp, instance
}
// Validate the constant pool indices and types
if (!cp->is_within_bounds(class_index) ||
- !cp->tag_at(class_index).is_klass_reference()) {
+ !is_klass_reference(cp, class_index)) {
classfile_parse_error("Invalid or out-of-bounds class index in EnclosingMethod attribute in class file %s", CHECK);
}
if (method_index != 0 &&
@@ -2349,6 +2465,7 @@ void ClassFileParser::java_lang_Class_fix_post(int* next_nonstatic_oop_offset_pt
instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
Handle class_loader,
Handle protection_domain,
+ GrowableArray* cp_patches,
symbolHandle& parsed_name,
TRAPS) {
// So that JVMTI can cache class file in the state before retransformable agents
@@ -2380,6 +2497,7 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
}
}
+ _cp_patches = cp_patches;
instanceKlassHandle nullHandle;
@@ -2510,14 +2628,22 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
CHECK_(nullHandle));
} else {
check_property(valid_cp_range(super_class_index, cp_size) &&
- cp->tag_at(super_class_index).is_unresolved_klass(),
+ is_klass_reference(cp, super_class_index),
"Invalid superclass index %u in class file %s",
super_class_index,
CHECK_(nullHandle));
// The class name should be legal because it is checked when parsing constant pool.
// However, make sure it is not an array type.
+ bool is_array = false;
+ if (cp->tag_at(super_class_index).is_klass()) {
+ super_klass = instanceKlassHandle(THREAD, cp->resolved_klass_at(super_class_index));
+ if (_need_verify)
+ is_array = super_klass->oop_is_array();
+ } else if (_need_verify) {
+ is_array = (cp->unresolved_klass_at(super_class_index)->byte_at(0) == JVM_SIGNATURE_ARRAY);
+ }
if (_need_verify) {
- guarantee_property(cp->unresolved_klass_at(super_class_index)->byte_at(0) != JVM_SIGNATURE_ARRAY,
+ guarantee_property(!is_array,
"Bad superclass name in class file %s", CHECK_(nullHandle));
}
}
@@ -2557,7 +2683,7 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
objArrayHandle methods_default_annotations(THREAD, methods_default_annotations_oop);
// We check super class after class file is parsed and format is checked
- if (super_class_index > 0) {
+ if (super_class_index > 0 && super_klass.is_null()) {
symbolHandle sk (THREAD, cp->klass_name_at(super_class_index));
if (access_flags.is_interface()) {
// Before attempting to resolve the superclass, check for class format
@@ -2574,6 +2700,10 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
CHECK_(nullHandle));
KlassHandle kh (THREAD, k);
super_klass = instanceKlassHandle(THREAD, kh());
+ if (LinkWellKnownClasses) // my super class is well known to me
+ cp->klass_at_put(super_class_index, super_klass()); // eagerly resolve
+ }
+ if (super_klass.not_null()) {
if (super_klass->is_interface()) {
ResourceMark rm(THREAD);
Exceptions::fthrow(
@@ -3000,6 +3130,8 @@ instanceKlassHandle ClassFileParser::parseClassFile(symbolHandle name,
this_klass->set_method_ordering(method_ordering());
this_klass->set_initial_method_idnum(methods->length());
this_klass->set_name(cp->klass_name_at(this_class_index));
+ if (LinkWellKnownClasses) // I am well known to myself
+ cp->klass_at_put(this_class_index, this_klass()); // eagerly resolve
this_klass->set_protection_domain(protection_domain());
this_klass->set_fields_annotations(fields_annotations());
this_klass->set_methods_annotations(methods_annotations());
diff --git a/hotspot/src/share/vm/classfile/classFileParser.hpp b/hotspot/src/share/vm/classfile/classFileParser.hpp
index a29905fc4c1..6cb79f11b69 100644
--- a/hotspot/src/share/vm/classfile/classFileParser.hpp
+++ b/hotspot/src/share/vm/classfile/classFileParser.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright 1997-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1997-2008 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@ class ClassFileParser VALUE_OBJ_CLASS_SPEC {
u2 _major_version;
u2 _minor_version;
symbolHandle _class_name;
+ GrowableArray* _cp_patches; // overrides for CP entries
bool _has_finalizer;
bool _has_empty_finalizer;
@@ -203,6 +204,35 @@ class ClassFileParser VALUE_OBJ_CLASS_SPEC {
char* skip_over_field_name(char* name, bool slash_ok, unsigned int length);
char* skip_over_field_signature(char* signature, bool void_ok, unsigned int length, TRAPS);
+ bool has_cp_patch_at(int index) {
+ assert(AnonymousClasses, "");
+ assert(index >= 0, "oob");
+ return (_cp_patches != NULL
+ && index < _cp_patches->length()
+ && _cp_patches->adr_at(index)->not_null());
+ }
+ Handle cp_patch_at(int index) {
+ assert(has_cp_patch_at(index), "oob");
+ return _cp_patches->at(index);
+ }
+ Handle clear_cp_patch_at(int index) {
+ Handle patch = cp_patch_at(index);
+ _cp_patches->at_put(index, Handle());
+ assert(!has_cp_patch_at(index), "");
+ return patch;
+ }
+ void patch_constant_pool(constantPoolHandle cp, int index, Handle patch, TRAPS);
+
+ // Wrapper for constantTag.is_klass_[or_]reference.
+ // In older versions of the VM, klassOops cannot sneak into early phases of
+ // constant pool construction, but in later versions they can.
+ // %%% Let's phase out the old is_klass_reference.
+ bool is_klass_reference(constantPoolHandle cp, int index) {
+ return ((LinkWellKnownClasses || AnonymousClasses)
+ ? cp->tag_at(index).is_klass_or_reference()
+ : cp->tag_at(index).is_klass_reference());
+ }
+
public:
// Constructor
ClassFileParser(ClassFileStream* st) { set_stream(st); }
@@ -218,6 +248,14 @@ class ClassFileParser VALUE_OBJ_CLASS_SPEC {
Handle class_loader,
Handle protection_domain,
symbolHandle& parsed_name,
+ TRAPS) {
+ return parseClassFile(name, class_loader, protection_domain, NULL, parsed_name, THREAD);
+ }
+ instanceKlassHandle parseClassFile(symbolHandle name,
+ Handle class_loader,
+ Handle protection_domain,
+ GrowableArray* cp_patches,
+ symbolHandle& parsed_name,
TRAPS);
// Verifier checks
diff --git a/hotspot/src/share/vm/classfile/systemDictionary.cpp b/hotspot/src/share/vm/classfile/systemDictionary.cpp
index 772716311a2..0908e1c8785 100644
--- a/hotspot/src/share/vm/classfile/systemDictionary.cpp
+++ b/hotspot/src/share/vm/classfile/systemDictionary.cpp
@@ -937,6 +937,8 @@ klassOop SystemDictionary::parse_stream(symbolHandle class_name,
Handle class_loader,
Handle protection_domain,
ClassFileStream* st,
+ KlassHandle host_klass,
+ GrowableArray* cp_patches,
TRAPS) {
symbolHandle parsed_name;
@@ -953,10 +955,10 @@ klassOop SystemDictionary::parse_stream(symbolHandle class_name,
instanceKlassHandle k = ClassFileParser(st).parseClassFile(class_name,
class_loader,
protection_domain,
+ cp_patches,
parsed_name,
THREAD);
-
// We don't redefine the class, so we just need to clean up whether there
// was an error or not (don't want to modify any system dictionary
// data structures).
@@ -973,6 +975,30 @@ klassOop SystemDictionary::parse_stream(symbolHandle class_name,
}
}
+ if (host_klass.not_null() && k.not_null()) {
+ assert(AnonymousClasses, "");
+ // If it's anonymous, initialize it now, since nobody else will.
+ k->set_host_klass(host_klass());
+
+ {
+ MutexLocker mu_r(Compile_lock, THREAD);
+
+ // Add to class hierarchy, initialize vtables, and do possible
+ // deoptimizations.
+ add_to_hierarchy(k, CHECK_NULL); // No exception, but can block
+
+ // But, do not add to system dictionary.
+ }
+
+ k->eager_initialize(THREAD);
+
+ // notify jvmti
+ if (JvmtiExport::should_post_class_load()) {
+ assert(THREAD->is_Java_thread(), "thread->is_Java_thread()");
+ JvmtiExport::post_class_load((JavaThread *) THREAD, k());
+ }
+ }
+
return k();
}
diff --git a/hotspot/src/share/vm/classfile/systemDictionary.hpp b/hotspot/src/share/vm/classfile/systemDictionary.hpp
index 38d27d84841..beade180fcd 100644
--- a/hotspot/src/share/vm/classfile/systemDictionary.hpp
+++ b/hotspot/src/share/vm/classfile/systemDictionary.hpp
@@ -228,6 +228,16 @@ public:
Handle class_loader,
Handle protection_domain,
ClassFileStream* st,
+ TRAPS) {
+ KlassHandle nullHandle;
+ return parse_stream(class_name, class_loader, protection_domain, st, nullHandle, NULL, THREAD);
+ }
+ static klassOop parse_stream(symbolHandle class_name,
+ Handle class_loader,
+ Handle protection_domain,
+ ClassFileStream* st,
+ KlassHandle host_klass,
+ GrowableArray* cp_patches,
TRAPS);
// Resolve from stream (called by jni_DefineClass and JVM_DefineClass)
diff --git a/hotspot/src/share/vm/classfile/verifier.cpp b/hotspot/src/share/vm/classfile/verifier.cpp
index 8373f9beaa0..179de12ea77 100644
--- a/hotspot/src/share/vm/classfile/verifier.cpp
+++ b/hotspot/src/share/vm/classfile/verifier.cpp
@@ -1600,7 +1600,11 @@ void ClassVerifier::verify_ldc(
types = (1 << JVM_CONSTANT_Double) | (1 << JVM_CONSTANT_Long);
verify_cp_type(index, cp, types, CHECK_VERIFY(this));
}
- if (tag.is_string() || tag.is_unresolved_string()) {
+ if (tag.is_string() && cp->is_pseudo_string_at(index)) {
+ current_frame->push_stack(
+ VerificationType::reference_type(
+ vmSymbols::java_lang_Object()), CHECK_VERIFY(this));
+ } else if (tag.is_string() || tag.is_unresolved_string()) {
current_frame->push_stack(
VerificationType::reference_type(
vmSymbols::java_lang_String()), CHECK_VERIFY(this));
diff --git a/hotspot/src/share/vm/code/nmethod.cpp b/hotspot/src/share/vm/code/nmethod.cpp
index 282a4876262..6baa28690db 100644
--- a/hotspot/src/share/vm/code/nmethod.cpp
+++ b/hotspot/src/share/vm/code/nmethod.cpp
@@ -1350,11 +1350,7 @@ bool nmethod::can_unload(BoolObjectClosure* is_alive,
return false;
}
}
- if (!UseParallelOldGC || !VerifyParallelOldWithMarkSweep) {
- // Cannot do this test if verification of the UseParallelOldGC
- // code using the PSMarkSweep code is being done.
- assert(unloading_occurred, "Inconsistency in unloading");
- }
+ assert(unloading_occurred, "Inconsistency in unloading");
make_unloaded(is_alive, obj);
return true;
}
diff --git a/hotspot/src/share/vm/compiler/methodLiveness.cpp b/hotspot/src/share/vm/compiler/methodLiveness.cpp
index 92d60cfeca8..a9a90a07197 100644
--- a/hotspot/src/share/vm/compiler/methodLiveness.cpp
+++ b/hotspot/src/share/vm/compiler/methodLiveness.cpp
@@ -76,8 +76,9 @@ class BitCounter: public BitMapClosure {
BitCounter() : _count(0) {}
// Callback when bit in map is set
- virtual void do_bit(size_t offset) {
+ virtual bool do_bit(size_t offset) {
_count++;
+ return true;
}
int count() {
@@ -467,7 +468,7 @@ MethodLivenessResult MethodLiveness::get_liveness_at(int entry_bci) {
bci = 0;
}
- MethodLivenessResult answer(NULL,0);
+ MethodLivenessResult answer((uintptr_t*)NULL,0);
if (_block_count > 0) {
if (TimeLivenessAnalysis) _time_total.start();
diff --git a/hotspot/src/share/vm/compiler/methodLiveness.hpp b/hotspot/src/share/vm/compiler/methodLiveness.hpp
index a679c34d007..705a36f762b 100644
--- a/hotspot/src/share/vm/compiler/methodLiveness.hpp
+++ b/hotspot/src/share/vm/compiler/methodLiveness.hpp
@@ -29,7 +29,7 @@ class MethodLivenessResult : public BitMap {
bool _is_valid;
public:
- MethodLivenessResult(uintptr_t* map, idx_t size_in_bits)
+ MethodLivenessResult(BitMap::bm_word_t* map, idx_t size_in_bits)
: BitMap(map, size_in_bits)
, _is_valid(false)
{}
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp
index 3115b6b1127..de5955d503e 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/cmsOopClosures.hpp
@@ -325,24 +325,30 @@ class Par_PushOrMarkClosure: public OopClosure {
// For objects in CMS generation, this closure marks
// given objects (transitively) as being reachable/live.
// This is currently used during the (weak) reference object
-// processing phase of the CMS final checkpoint step.
+// processing phase of the CMS final checkpoint step, as
+// well as during the concurrent precleaning of the discovered
+// reference lists.
class CMSKeepAliveClosure: public OopClosure {
private:
CMSCollector* _collector;
const MemRegion _span;
CMSMarkStack* _mark_stack;
CMSBitMap* _bit_map;
+ bool _concurrent_precleaning;
protected:
DO_OOP_WORK_DEFN
public:
CMSKeepAliveClosure(CMSCollector* collector, MemRegion span,
- CMSBitMap* bit_map, CMSMarkStack* mark_stack):
+ CMSBitMap* bit_map, CMSMarkStack* mark_stack,
+ bool cpc):
_collector(collector),
_span(span),
_bit_map(bit_map),
- _mark_stack(mark_stack) {
+ _mark_stack(mark_stack),
+ _concurrent_precleaning(cpc) {
assert(!_span.is_empty(), "Empty span could spell trouble");
}
+ bool concurrent_precleaning() const { return _concurrent_precleaning; }
virtual void do_oop(oop* p);
virtual void do_oop(narrowOop* p);
inline void do_oop_nv(oop* p) { CMSKeepAliveClosure::do_oop_work(p); }
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp
index af9a9ecf3dc..4b1fecd7cb3 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp
@@ -790,7 +790,7 @@ CompactibleFreeListSpace::object_iterate_careful_m(MemRegion mr,
}
-HeapWord* CompactibleFreeListSpace::block_start(const void* p) const {
+HeapWord* CompactibleFreeListSpace::block_start_const(const void* p) const {
NOT_PRODUCT(verify_objects_initialized());
return _bt.block_start(p);
}
@@ -2286,9 +2286,9 @@ void CompactibleFreeListSpace::verifyIndexedFreeLists() const {
}
void CompactibleFreeListSpace::verifyIndexedFreeList(size_t size) const {
- guarantee(size % 2 == 0, "Odd slots should be empty");
- for (FreeChunk* fc = _indexedFreeList[size].head(); fc != NULL;
- fc = fc->next()) {
+ FreeChunk* fc = _indexedFreeList[size].head();
+ guarantee((size % 2 == 0) || fc == NULL, "Odd slots should be empty");
+ for (; fc != NULL; fc = fc->next()) {
guarantee(fc->size() == size, "Size inconsistency");
guarantee(fc->isFree(), "!free?");
guarantee(fc->next() == NULL || fc->next()->prev() == fc, "Broken list");
@@ -2790,10 +2790,11 @@ initialize_sequential_subtasks_for_rescan(int n_threads) {
assert(n_threads > 0, "Unexpected n_threads argument");
const size_t task_size = rescan_task_size();
size_t n_tasks = (used_region().word_size() + task_size - 1)/task_size;
- assert((used_region().start() + (n_tasks - 1)*task_size <
- used_region().end()) &&
- (used_region().start() + n_tasks*task_size >=
- used_region().end()), "n_task calculation incorrect");
+ assert((n_tasks == 0) == used_region().is_empty(), "n_tasks incorrect");
+ assert(n_tasks == 0 ||
+ ((used_region().start() + (n_tasks - 1)*task_size < used_region().end()) &&
+ (used_region().start() + n_tasks*task_size >= used_region().end())),
+ "n_tasks calculation incorrect");
SequentialSubTasksDone* pst = conc_par_seq_tasks();
assert(!pst->valid(), "Clobbering existing data?");
pst->set_par_threads(n_threads);
@@ -2833,7 +2834,7 @@ initialize_sequential_subtasks_for_marking(int n_threads,
assert(n_tasks == 0 ||
((span.start() + (n_tasks - 1)*task_size < span.end()) &&
(span.start() + n_tasks*task_size >= span.end())),
- "n_task calculation incorrect");
+ "n_tasks calculation incorrect");
SequentialSubTasksDone* pst = conc_par_seq_tasks();
assert(!pst->valid(), "Clobbering existing data?");
pst->set_par_threads(n_threads);
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp
index 9ac7b03d40d..5306a8f3085 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp
@@ -502,7 +502,7 @@ class CompactibleFreeListSpace: public CompactibleSpace {
void blk_iterate(BlkClosure* cl);
void blk_iterate_careful(BlkClosureCareful* cl);
- HeapWord* block_start(const void* p) const;
+ HeapWord* block_start_const(const void* p) const;
HeapWord* block_start_careful(const void* p) const;
size_t block_size(const HeapWord* p) const;
size_t block_size_no_stall(HeapWord* p, const CMSCollector* c) const;
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
index 697ce756825..e8344107659 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp
@@ -538,6 +538,7 @@ CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,
_survivor_chunk_capacity(0), // -- ditto --
_survivor_chunk_index(0), // -- ditto --
_ser_pmc_preclean_ovflw(0),
+ _ser_kac_preclean_ovflw(0),
_ser_pmc_remark_ovflw(0),
_par_pmc_remark_ovflw(0),
_ser_kac_ovflw(0),
@@ -1960,6 +1961,7 @@ void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {
ref_processor()->set_enqueuing_is_done(false);
ref_processor()->enable_discovery();
+ ref_processor()->setup_policy(clear_all_soft_refs);
// If an asynchronous collection finishes, the _modUnionTable is
// all clear. If we are assuming the collection from an asynchronous
// collection, clear the _modUnionTable.
@@ -2383,6 +2385,9 @@ void CMSCollector::collect_in_foreground(bool clear_all_soft_refs) {
Universe::verify(true);
}
+ // Snapshot the soft reference policy to be used in this collection cycle.
+ ref_processor()->setup_policy(clear_all_soft_refs);
+
bool init_mark_was_synchronous = false; // until proven otherwise
while (_collectorState != Idling) {
if (TraceCMSState) {
@@ -2761,13 +2766,14 @@ class VerifyMarkedClosure: public BitMapClosure {
public:
VerifyMarkedClosure(CMSBitMap* bm): _marks(bm), _failed(false) {}
- void do_bit(size_t offset) {
+ bool do_bit(size_t offset) {
HeapWord* addr = _marks->offsetToHeapWord(offset);
if (!_marks->isMarked(addr)) {
oop(addr)->print();
gclog_or_tty->print_cr(" ("INTPTR_FORMAT" should have been marked)", addr);
_failed = true;
}
+ return true;
}
bool failed() { return _failed; }
@@ -3650,6 +3656,7 @@ class CMSConcMarkingTask: public YieldingFlexibleGangTask {
CompactibleFreeListSpace* _cms_space;
CompactibleFreeListSpace* _perm_space;
HeapWord* _global_finger;
+ HeapWord* _restart_addr;
// Exposed here for yielding support
Mutex* const _bit_map_lock;
@@ -3680,7 +3687,7 @@ class CMSConcMarkingTask: public YieldingFlexibleGangTask {
_term.set_task(this);
assert(_cms_space->bottom() < _perm_space->bottom(),
"Finger incorrectly initialized below");
- _global_finger = _cms_space->bottom();
+ _restart_addr = _global_finger = _cms_space->bottom();
}
@@ -3698,6 +3705,10 @@ class CMSConcMarkingTask: public YieldingFlexibleGangTask {
bool result() { return _result; }
void reset(HeapWord* ra) {
+ assert(_global_finger >= _cms_space->end(), "Postcondition of ::work(i)");
+ assert(_global_finger >= _perm_space->end(), "Postcondition of ::work(i)");
+ assert(ra < _perm_space->end(), "ra too large");
+ _restart_addr = _global_finger = ra;
_term.reset_for_reuse();
}
@@ -3842,16 +3853,24 @@ void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
int n_tasks = pst->n_tasks();
// We allow that there may be no tasks to do here because
// we are restarting after a stack overflow.
- assert(pst->valid() || n_tasks == 0, "Uninitializd use?");
+ assert(pst->valid() || n_tasks == 0, "Uninitialized use?");
int nth_task = 0;
- HeapWord* start = sp->bottom();
+ HeapWord* aligned_start = sp->bottom();
+ if (sp->used_region().contains(_restart_addr)) {
+ // Align down to a card boundary for the start of 0th task
+ // for this space.
+ aligned_start =
+ (HeapWord*)align_size_down((uintptr_t)_restart_addr,
+ CardTableModRefBS::card_size);
+ }
+
size_t chunk_size = sp->marking_task_size();
while (!pst->is_task_claimed(/* reference */ nth_task)) {
// Having claimed the nth task in this space,
// compute the chunk that it corresponds to:
- MemRegion span = MemRegion(start + nth_task*chunk_size,
- start + (nth_task+1)*chunk_size);
+ MemRegion span = MemRegion(aligned_start + nth_task*chunk_size,
+ aligned_start + (nth_task+1)*chunk_size);
// Try and bump the global finger via a CAS;
// note that we need to do the global finger bump
// _before_ taking the intersection below, because
@@ -3866,26 +3885,40 @@ void CMSConcMarkingTask::do_scan_and_mark(int i, CompactibleFreeListSpace* sp) {
// beyond the "top" address of the space.
span = span.intersection(sp->used_region());
if (!span.is_empty()) { // Non-null task
- // We want to skip the first object because
- // the protocol is to scan any object in its entirety
- // that _starts_ in this span; a fortiori, any
- // object starting in an earlier span is scanned
- // as part of an earlier claimed task.
- // Below we use the "careful" version of block_start
- // so we do not try to navigate uninitialized objects.
- HeapWord* prev_obj = sp->block_start_careful(span.start());
- // Below we use a variant of block_size that uses the
- // Printezis bits to avoid waiting for allocated
- // objects to become initialized/parsable.
- while (prev_obj < span.start()) {
- size_t sz = sp->block_size_no_stall(prev_obj, _collector);
- if (sz > 0) {
- prev_obj += sz;
+ HeapWord* prev_obj;
+ assert(!span.contains(_restart_addr) || nth_task == 0,
+ "Inconsistency");
+ if (nth_task == 0) {
+ // For the 0th task, we'll not need to compute a block_start.
+ if (span.contains(_restart_addr)) {
+ // In the case of a restart because of stack overflow,
+ // we might additionally skip a chunk prefix.
+ prev_obj = _restart_addr;
} else {
- // In this case we may end up doing a bit of redundant
- // scanning, but that appears unavoidable, short of
- // locking the free list locks; see bug 6324141.
- break;
+ prev_obj = span.start();
+ }
+ } else {
+ // We want to skip the first object because
+ // the protocol is to scan any object in its entirety
+ // that _starts_ in this span; a fortiori, any
+ // object starting in an earlier span is scanned
+ // as part of an earlier claimed task.
+ // Below we use the "careful" version of block_start
+ // so we do not try to navigate uninitialized objects.
+ prev_obj = sp->block_start_careful(span.start());
+ // Below we use a variant of block_size that uses the
+ // Printezis bits to avoid waiting for allocated
+ // objects to become initialized/parsable.
+ while (prev_obj < span.start()) {
+ size_t sz = sp->block_size_no_stall(prev_obj, _collector);
+ if (sz > 0) {
+ prev_obj += sz;
+ } else {
+ // In this case we may end up doing a bit of redundant
+ // scanning, but that appears unavoidable, short of
+ // locking the free list locks; see bug 6324141.
+ break;
+ }
}
}
if (prev_obj < span.end()) {
@@ -3938,12 +3971,14 @@ class Par_ConcMarkingClosure: public OopClosure {
void handle_stack_overflow(HeapWord* lost);
};
-// Grey object rescan during work stealing phase --
-// the salient assumption here is that stolen oops must
-// always be initialized, so we do not need to check for
-// uninitialized objects before scanning here.
+// Grey object scanning during work stealing phase --
+// the salient assumption here is that any references
+// that are in these stolen objects being scanned must
+// already have been initialized (else they would not have
+// been published), so we do not need to check for
+// uninitialized objects before pushing here.
void Par_ConcMarkingClosure::do_oop(oop obj) {
- assert(obj->is_oop_or_null(), "expected an oop or NULL");
+ assert(obj->is_oop_or_null(true), "expected an oop or NULL");
HeapWord* addr = (HeapWord*)obj;
// Check if oop points into the CMS generation
// and is not marked
@@ -4001,7 +4036,7 @@ void Par_ConcMarkingClosure::trim_queue(size_t max) {
// in CMSCollector's _restart_address.
void Par_ConcMarkingClosure::handle_stack_overflow(HeapWord* lost) {
// We need to do this under a mutex to prevent other
- // workers from interfering with the expansion below.
+ // workers from interfering with the work done below.
MutexLockerEx ml(_overflow_stack->par_lock(),
Mutex::_no_safepoint_check_flag);
// Remember the least grey address discarded
@@ -4358,10 +4393,10 @@ size_t CMSCollector::preclean_work(bool clean_refs, bool clean_survivor) {
CMSPrecleanRefsYieldClosure yield_cl(this);
assert(rp->span().equals(_span), "Spans should be equal");
CMSKeepAliveClosure keep_alive(this, _span, &_markBitMap,
- &_markStack);
+ &_markStack, true /* preclean */);
CMSDrainMarkingStackClosure complete_trace(this,
- _span, &_markBitMap, &_markStack,
- &keep_alive);
+ _span, &_markBitMap, &_markStack,
+ &keep_alive, true /* preclean */);
// We don't want this step to interfere with a young
// collection because we don't want to take CPU
@@ -4560,11 +4595,11 @@ size_t CMSCollector::preclean_mod_union_table(
if (!dirtyRegion.is_empty()) {
assert(numDirtyCards > 0, "consistency check");
HeapWord* stop_point = NULL;
+ stopTimer();
+ CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
+ bitMapLock());
+ startTimer();
{
- stopTimer();
- CMSTokenSyncWithLocks ts(true, gen->freelistLock(),
- bitMapLock());
- startTimer();
verify_work_stacks_empty();
verify_overflow_empty();
sample_eden();
@@ -4581,10 +4616,6 @@ size_t CMSCollector::preclean_mod_union_table(
assert((CMSPermGenPrecleaningEnabled && (gen == _permGen)) ||
(_collectorState == AbortablePreclean && should_abort_preclean()),
"Unparsable objects should only be in perm gen.");
-
- stopTimer();
- CMSTokenSyncWithLocks ts(true, bitMapLock());
- startTimer();
_modUnionTable.mark_range(MemRegion(stop_point, dirtyRegion.end()));
if (should_abort_preclean()) {
break; // out of preclean loop
@@ -4640,8 +4671,11 @@ size_t CMSCollector::preclean_card_table(ConcurrentMarkSweepGeneration* gen,
startTimer();
sample_eden();
// Get and clear dirty region from card table
- dirtyRegion = _ct->ct_bs()->dirty_card_range_after_preclean(
- MemRegion(nextAddr, endAddr));
+ dirtyRegion = _ct->ct_bs()->dirty_card_range_after_reset(
+ MemRegion(nextAddr, endAddr),
+ true,
+ CardTableModRefBS::precleaned_card_val());
+
assert(dirtyRegion.start() >= nextAddr,
"returned region inconsistent?");
}
@@ -4819,17 +4853,19 @@ void CMSCollector::checkpointRootsFinalWork(bool asynch,
// recurrence of that condition.
assert(_markStack.isEmpty(), "No grey objects");
size_t ser_ovflw = _ser_pmc_remark_ovflw + _ser_pmc_preclean_ovflw +
- _ser_kac_ovflw;
+ _ser_kac_ovflw + _ser_kac_preclean_ovflw;
if (ser_ovflw > 0) {
if (PrintCMSStatistics != 0) {
gclog_or_tty->print_cr("Marking stack overflow (benign) "
- "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT")",
+ "(pmc_pc="SIZE_FORMAT", pmc_rm="SIZE_FORMAT", kac="SIZE_FORMAT
+ ", kac_preclean="SIZE_FORMAT")",
_ser_pmc_preclean_ovflw, _ser_pmc_remark_ovflw,
- _ser_kac_ovflw);
+ _ser_kac_ovflw, _ser_kac_preclean_ovflw);
}
_markStack.expand();
_ser_pmc_remark_ovflw = 0;
_ser_pmc_preclean_ovflw = 0;
+ _ser_kac_preclean_ovflw = 0;
_ser_kac_ovflw = 0;
}
if (_par_pmc_remark_ovflw > 0 || _par_kac_ovflw > 0) {
@@ -5409,8 +5445,8 @@ void CMSCollector::do_remark_non_parallel() {
&mrias_cl);
{
TraceTime t("grey object rescan", PrintGCDetails, false, gclog_or_tty);
- // Iterate over the dirty cards, marking them precleaned, and
- // setting the corresponding bits in the mod union table.
+ // Iterate over the dirty cards, setting the corresponding bits in the
+ // mod union table.
{
ModUnionClosure modUnionClosure(&_modUnionTable);
_ct->ct_bs()->dirty_card_iterate(
@@ -5642,40 +5678,29 @@ void CMSCollector::refProcessingWork(bool asynch, bool clear_all_soft_refs) {
ResourceMark rm;
HandleMark hm;
- ReferencePolicy* soft_ref_policy;
-
- assert(!ref_processor()->enqueuing_is_done(), "Enqueuing should not be complete");
- // Process weak references.
- if (clear_all_soft_refs) {
- soft_ref_policy = new AlwaysClearPolicy();
- } else {
-#ifdef COMPILER2
- soft_ref_policy = new LRUMaxHeapPolicy();
-#else
- soft_ref_policy = new LRUCurrentHeapPolicy();
-#endif // COMPILER2
- }
- verify_work_stacks_empty();
ReferenceProcessor* rp = ref_processor();
assert(rp->span().equals(_span), "Spans should be equal");
+ assert(!rp->enqueuing_is_done(), "Enqueuing should not be complete");
+ // Process weak references.
+ rp->setup_policy(clear_all_soft_refs);
+ verify_work_stacks_empty();
+
CMSKeepAliveClosure cmsKeepAliveClosure(this, _span, &_markBitMap,
- &_markStack);
+ &_markStack, false /* !preclean */);
CMSDrainMarkingStackClosure cmsDrainMarkingStackClosure(this,
_span, &_markBitMap, &_markStack,
- &cmsKeepAliveClosure);
+ &cmsKeepAliveClosure, false /* !preclean */);
{
TraceTime t("weak refs processing", PrintGCDetails, false, gclog_or_tty);
if (rp->processing_is_mt()) {
CMSRefProcTaskExecutor task_executor(*this);
- rp->process_discovered_references(soft_ref_policy,
- &_is_alive_closure,
+ rp->process_discovered_references(&_is_alive_closure,
&cmsKeepAliveClosure,
&cmsDrainMarkingStackClosure,
&task_executor);
} else {
- rp->process_discovered_references(soft_ref_policy,
- &_is_alive_closure,
+ rp->process_discovered_references(&_is_alive_closure,
&cmsKeepAliveClosure,
&cmsDrainMarkingStackClosure,
NULL);
@@ -6130,8 +6155,8 @@ void CMSCollector::verify_ok_to_terminate() const {
#endif
size_t CMSCollector::block_size_using_printezis_bits(HeapWord* addr) const {
- assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
- "missing Printezis mark?");
+ assert(_markBitMap.isMarked(addr) && _markBitMap.isMarked(addr + 1),
+ "missing Printezis mark?");
HeapWord* nextOneAddr = _markBitMap.getNextMarkedWordAddress(addr + 2);
size_t size = pointer_delta(nextOneAddr + 1, addr);
assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
@@ -6182,7 +6207,7 @@ HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
// bit vector itself. That is done by a separate call CMSBitMap::allocate()
// further below.
CMSBitMap::CMSBitMap(int shifter, int mutex_rank, const char* mutex_name):
- _bm(NULL,0),
+ _bm(),
_shifter(shifter),
_lock(mutex_rank >= 0 ? new Mutex(mutex_rank, mutex_name, true) : NULL)
{
@@ -6207,7 +6232,7 @@ bool CMSBitMap::allocate(MemRegion mr) {
}
assert(_virtual_space.committed_size() == brs.size(),
"didn't reserve backing store for all of CMS bit map?");
- _bm.set_map((uintptr_t*)_virtual_space.low());
+ _bm.set_map((BitMap::bm_word_t*)_virtual_space.low());
assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
_bmWordSize, "inconsistency in bit map sizing");
_bm.set_size(_bmWordSize >> _shifter);
@@ -6554,7 +6579,7 @@ void Par_MarkRefsIntoAndScanClosure::do_oop(oop obj) {
if (obj != NULL) {
// Ignore mark word because this could be an already marked oop
// that may be chained at the end of the overflow list.
- assert(obj->is_oop(), "expected an oop");
+ assert(obj->is_oop(true), "expected an oop");
HeapWord* addr = (HeapWord*)obj;
if (_span.contains(addr) &&
!_bit_map->isMarked(addr)) {
@@ -6845,10 +6870,10 @@ void MarkFromRootsClosure::reset(HeapWord* addr) {
// Should revisit to see if this should be restructured for
// greater efficiency.
-void MarkFromRootsClosure::do_bit(size_t offset) {
+bool MarkFromRootsClosure::do_bit(size_t offset) {
if (_skipBits > 0) {
_skipBits--;
- return;
+ return true;
}
// convert offset into a HeapWord*
HeapWord* addr = _bitMap->startWord() + offset;
@@ -6886,10 +6911,11 @@ void MarkFromRootsClosure::do_bit(size_t offset) {
} // ...else the setting of klass will dirty the card anyway.
}
DEBUG_ONLY(})
- return;
+ return true;
}
}
scanOopsInOop(addr);
+ return true;
}
// We take a break if we've been at this for a while,
@@ -7023,10 +7049,10 @@ Par_MarkFromRootsClosure::Par_MarkFromRootsClosure(CMSConcMarkingTask* task,
// Should revisit to see if this should be restructured for
// greater efficiency.
-void Par_MarkFromRootsClosure::do_bit(size_t offset) {
+bool Par_MarkFromRootsClosure::do_bit(size_t offset) {
if (_skip_bits > 0) {
_skip_bits--;
- return;
+ return true;
}
// convert offset into a HeapWord*
HeapWord* addr = _bit_map->startWord() + offset;
@@ -7041,10 +7067,11 @@ void Par_MarkFromRootsClosure::do_bit(size_t offset) {
if (p->klass_or_null() == NULL || !p->is_parsable()) {
// in the case of Clean-on-Enter optimization, redirty card
// and avoid clearing card by increasing the threshold.
- return;
+ return true;
}
}
scan_oops_in_oop(addr);
+ return true;
}
void Par_MarkFromRootsClosure::scan_oops_in_oop(HeapWord* ptr) {
@@ -7167,7 +7194,7 @@ void MarkFromRootsVerifyClosure::reset(HeapWord* addr) {
// Should revisit to see if this should be restructured for
// greater efficiency.
-void MarkFromRootsVerifyClosure::do_bit(size_t offset) {
+bool MarkFromRootsVerifyClosure::do_bit(size_t offset) {
// convert offset into a HeapWord*
HeapWord* addr = _verification_bm->startWord() + offset;
assert(_verification_bm->endWord() && addr < _verification_bm->endWord(),
@@ -7195,6 +7222,7 @@ void MarkFromRootsVerifyClosure::do_bit(size_t offset) {
new_oop->oop_iterate(&_pam_verify_closure);
}
assert(_mark_stack->isEmpty(), "tautology, emphasizing post-condition");
+ return true;
}
PushAndMarkVerifyClosure::PushAndMarkVerifyClosure(
@@ -7289,6 +7317,8 @@ Par_PushOrMarkClosure::Par_PushOrMarkClosure(CMSCollector* collector,
_should_remember_klasses(collector->should_unload_classes())
{ }
+// Assumes thread-safe access by callers, who are
+// responsible for mutual exclusion.
void CMSCollector::lower_restart_addr(HeapWord* low) {
assert(_span.contains(low), "Out of bounds addr");
if (_restart_addr == NULL) {
@@ -7314,7 +7344,7 @@ void PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
// in CMSCollector's _restart_address.
void Par_PushOrMarkClosure::handle_stack_overflow(HeapWord* lost) {
// We need to do this under a mutex to prevent other
- // workers from interfering with the expansion below.
+ // workers from interfering with the work done below.
MutexLockerEx ml(_overflow_stack->par_lock(),
Mutex::_no_safepoint_check_flag);
// Remember the least grey address discarded
@@ -7438,8 +7468,12 @@ PushAndMarkClosure::PushAndMarkClosure(CMSCollector* collector,
// Grey object rescan during pre-cleaning and second checkpoint phases --
// the non-parallel version (the parallel version appears further below.)
void PushAndMarkClosure::do_oop(oop obj) {
- // If _concurrent_precleaning, ignore mark word verification
- assert(obj->is_oop_or_null(_concurrent_precleaning),
+ // Ignore mark word verification. If during concurrent precleaning,
+ // the object monitor may be locked. If during the checkpoint
+ // phases, the object may already have been reached by a different
+ // path and may be at the end of the global overflow list (so
+ // the mark word may be NULL).
+ assert(obj->is_oop_or_null(true /* ignore mark word */),
"expected an oop or NULL");
HeapWord* addr = (HeapWord*)obj;
// Check if oop points into the CMS generation
@@ -8260,8 +8294,29 @@ void CMSKeepAliveClosure::do_oop(oop obj) {
}
)
if (simulate_overflow || !_mark_stack->push(obj)) {
- _collector->push_on_overflow_list(obj);
- _collector->_ser_kac_ovflw++;
+ if (_concurrent_precleaning) {
+ // We dirty the overflown object and let the remark
+ // phase deal with it.
+ assert(_collector->overflow_list_is_empty(), "Error");
+ // In the case of object arrays, we need to dirty all of
+ // the cards that the object spans. No locking or atomics
+ // are needed since no one else can be mutating the mod union
+ // table.
+ if (obj->is_objArray()) {
+ size_t sz = obj->size();
+ HeapWord* end_card_addr =
+ (HeapWord*)round_to((intptr_t)(addr+sz), CardTableModRefBS::card_size);
+ MemRegion redirty_range = MemRegion(addr, end_card_addr);
+ assert(!redirty_range.is_empty(), "Arithmetical tautology");
+ _collector->_modUnionTable.mark_range(redirty_range);
+ } else {
+ _collector->_modUnionTable.mark(addr);
+ }
+ _collector->_ser_kac_preclean_ovflw++;
+ } else {
+ _collector->push_on_overflow_list(obj);
+ _collector->_ser_kac_ovflw++;
+ }
}
}
}
@@ -8358,6 +8413,8 @@ const char* CMSExpansionCause::to_string(CMSExpansionCause::Cause cause) {
void CMSDrainMarkingStackClosure::do_void() {
// the max number to take from overflow list at a time
const size_t num = _mark_stack->capacity()/4;
+ assert(!_concurrent_precleaning || _collector->overflow_list_is_empty(),
+ "Overflow list should be NULL during concurrent phases");
while (!_mark_stack->isEmpty() ||
// if stack is empty, check the overflow list
_collector->take_from_overflow_list(num, _mark_stack)) {
diff --git a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
index 672bb8a8da7..44ef14e8db3 100644
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp
@@ -592,6 +592,7 @@ class CMSCollector: public CHeapObj {
size_t _ser_pmc_preclean_ovflw;
size_t _ser_pmc_remark_ovflw;
size_t _par_pmc_remark_ovflw;
+ size_t _ser_kac_preclean_ovflw;
size_t _ser_kac_ovflw;
size_t _par_kac_ovflw;
NOT_PRODUCT(size_t _num_par_pushes;)
@@ -1327,7 +1328,7 @@ class MarkFromRootsClosure: public BitMapClosure {
CMSMarkStack* markStack,
CMSMarkStack* revisitStack,
bool should_yield, bool verifying = false);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
void reset(HeapWord* addr);
inline void do_yield_check();
@@ -1363,7 +1364,7 @@ class Par_MarkFromRootsClosure: public BitMapClosure {
CMSMarkStack* overflow_stack,
CMSMarkStack* revisit_stack,
bool should_yield);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
inline void do_yield_check();
private:
@@ -1411,7 +1412,7 @@ class MarkFromRootsVerifyClosure: public BitMapClosure {
CMSBitMap* verification_bm,
CMSBitMap* cms_bm,
CMSMarkStack* mark_stack);
- void do_bit(size_t offset);
+ bool do_bit(size_t offset);
void reset(HeapWord* addr);
};
@@ -1420,8 +1421,9 @@ class MarkFromRootsVerifyClosure: public BitMapClosure {
// "empty" (i.e. the bit vector doesn't have any 1-bits).
class FalseBitMapClosure: public BitMapClosure {
public:
- void do_bit(size_t offset) {
+ bool do_bit(size_t offset) {
guarantee(false, "Should not have a 1 bit");
+ return true;
}
};
@@ -1748,21 +1750,30 @@ class SweepClosure: public BlkClosureCareful {
// work-routine/closure used to complete transitive
// marking of objects as live after a certain point
// in which an initial set has been completely accumulated.
+// This closure is currently used both during the final
+// remark stop-world phase, as well as during the concurrent
+// precleaning of the discovered reference lists.
class CMSDrainMarkingStackClosure: public VoidClosure {
CMSCollector* _collector;
MemRegion _span;
CMSMarkStack* _mark_stack;
CMSBitMap* _bit_map;
CMSKeepAliveClosure* _keep_alive;
+ bool _concurrent_precleaning;
public:
CMSDrainMarkingStackClosure(CMSCollector* collector, MemRegion span,
CMSBitMap* bit_map, CMSMarkStack* mark_stack,
- CMSKeepAliveClosure* keep_alive):
+ CMSKeepAliveClosure* keep_alive,
+ bool cpc):
_collector(collector),
_span(span),
_bit_map(bit_map),
_mark_stack(mark_stack),
- _keep_alive(keep_alive) { }
+ _keep_alive(keep_alive),
+ _concurrent_precleaning(cpc) {
+ assert(_concurrent_precleaning == _keep_alive->concurrent_precleaning(),
+ "Mismatch");
+ }
void do_void();
};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp b/hotspot/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp
new file mode 100644
index 00000000000..1124e5d799f
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/bufferingOopClosure.hpp
@@ -0,0 +1,195 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// A BufferingOops closure tries to separate out the cost of finding roots
+// from the cost of applying closures to them. It maintains an array of
+// ref-containing locations. Until the array is full, applying the closure
+// to an oop* merely records that location in the array. Since this
+// closure app cost is small, an elapsed timer can approximately attribute
+// all of this cost to the cost of finding the roots. When the array fills
+// up, the wrapped closure is applied to all elements, keeping track of
+// this elapsed time of this process, and leaving the array empty.
+// The caller must be sure to call "done" to process any unprocessed
+// buffered entriess.
+
+class Generation;
+class HeapRegion;
+
+class BufferingOopClosure: public OopClosure {
+protected:
+ enum PrivateConstants {
+ BufferLength = 1024
+ };
+
+ oop *_buffer[BufferLength];
+ oop **_buffer_top;
+ oop **_buffer_curr;
+
+ OopClosure *_oc;
+ double _closure_app_seconds;
+
+ void process_buffer () {
+
+ double start = os::elapsedTime();
+ for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
+ _oc->do_oop(*curr);
+ }
+ _buffer_curr = _buffer;
+ _closure_app_seconds += (os::elapsedTime() - start);
+ }
+
+public:
+ virtual void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+ virtual void do_oop(oop *p) {
+ if (_buffer_curr == _buffer_top) {
+ process_buffer();
+ }
+
+ *_buffer_curr = p;
+ ++_buffer_curr;
+ }
+ void done () {
+ if (_buffer_curr > _buffer) {
+ process_buffer();
+ }
+ }
+ double closure_app_seconds () {
+ return _closure_app_seconds;
+ }
+ BufferingOopClosure (OopClosure *oc) :
+ _oc(oc),
+ _buffer_curr(_buffer), _buffer_top(_buffer + BufferLength),
+ _closure_app_seconds(0.0) { }
+};
+
+class BufferingOopsInGenClosure: public OopsInGenClosure {
+ BufferingOopClosure _boc;
+ OopsInGenClosure* _oc;
+public:
+ BufferingOopsInGenClosure(OopsInGenClosure *oc) :
+ _boc(oc), _oc(oc) {}
+
+ virtual void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ virtual void do_oop(oop* p) {
+ assert(generation()->is_in_reserved(p), "Must be in!");
+ _boc.do_oop(p);
+ }
+
+ void done() {
+ _boc.done();
+ }
+
+ double closure_app_seconds () {
+ return _boc.closure_app_seconds();
+ }
+
+ void set_generation(Generation* gen) {
+ OopsInGenClosure::set_generation(gen);
+ _oc->set_generation(gen);
+ }
+
+ void reset_generation() {
+ // Make sure we finish the current work with the current generation.
+ _boc.done();
+ OopsInGenClosure::reset_generation();
+ _oc->reset_generation();
+ }
+
+};
+
+
+class BufferingOopsInHeapRegionClosure: public OopsInHeapRegionClosure {
+private:
+ enum PrivateConstants {
+ BufferLength = 1024
+ };
+
+ oop *_buffer[BufferLength];
+ oop **_buffer_top;
+ oop **_buffer_curr;
+
+ HeapRegion *_hr_buffer[BufferLength];
+ HeapRegion **_hr_curr;
+
+ OopsInHeapRegionClosure *_oc;
+ double _closure_app_seconds;
+
+ void process_buffer () {
+
+ assert((_hr_curr - _hr_buffer) == (_buffer_curr - _buffer),
+ "the two lengths should be the same");
+
+ double start = os::elapsedTime();
+ HeapRegion **hr_curr = _hr_buffer;
+ HeapRegion *hr_prev = NULL;
+ for (oop **curr = _buffer; curr < _buffer_curr; ++curr) {
+ HeapRegion *region = *hr_curr;
+ if (region != hr_prev) {
+ _oc->set_region(region);
+ hr_prev = region;
+ }
+ _oc->do_oop(*curr);
+ ++hr_curr;
+ }
+ _buffer_curr = _buffer;
+ _hr_curr = _hr_buffer;
+ _closure_app_seconds += (os::elapsedTime() - start);
+ }
+
+public:
+ virtual void do_oop(narrowOop *p) {
+ guarantee(false, "NYI");
+ }
+
+ virtual void do_oop(oop *p) {
+ if (_buffer_curr == _buffer_top) {
+ assert(_hr_curr > _hr_buffer, "_hr_curr should be consistent with _buffer_curr");
+ process_buffer();
+ }
+
+ *_buffer_curr = p;
+ ++_buffer_curr;
+ *_hr_curr = _from;
+ ++_hr_curr;
+ }
+ void done () {
+ if (_buffer_curr > _buffer) {
+ assert(_hr_curr > _hr_buffer, "_hr_curr should be consistent with _buffer_curr");
+ process_buffer();
+ }
+ }
+ double closure_app_seconds () {
+ return _closure_app_seconds;
+ }
+ BufferingOopsInHeapRegionClosure (OopsInHeapRegionClosure *oc) :
+ _oc(oc),
+ _buffer_curr(_buffer), _buffer_top(_buffer + BufferLength),
+ _hr_curr(_hr_buffer),
+ _closure_app_seconds(0.0) { }
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp b/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp
new file mode 100644
index 00000000000..fbc5f4f151b
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.cpp
@@ -0,0 +1,409 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_collectionSetChooser.cpp.incl"
+
+CSetChooserCache::CSetChooserCache() {
+ for (int i = 0; i < CacheLength; ++i)
+ _cache[i] = NULL;
+ clear();
+}
+
+void CSetChooserCache::clear() {
+ _occupancy = 0;
+ _first = 0;
+ for (int i = 0; i < CacheLength; ++i) {
+ HeapRegion *hr = _cache[i];
+ if (hr != NULL)
+ hr->set_sort_index(-1);
+ _cache[i] = NULL;
+ }
+}
+
+#ifndef PRODUCT
+bool CSetChooserCache::verify() {
+ int index = _first;
+ HeapRegion *prev = NULL;
+ for (int i = 0; i < _occupancy; ++i) {
+ guarantee(_cache[index] != NULL, "cache entry should not be empty");
+ HeapRegion *hr = _cache[index];
+ guarantee(!hr->is_young(), "should not be young!");
+ if (prev != NULL) {
+ guarantee(prev->gc_efficiency() >= hr->gc_efficiency(),
+ "cache should be correctly ordered");
+ }
+ guarantee(hr->sort_index() == get_sort_index(index),
+ "sort index should be correct");
+ index = trim_index(index + 1);
+ prev = hr;
+ }
+
+ for (int i = 0; i < (CacheLength - _occupancy); ++i) {
+ guarantee(_cache[index] == NULL, "cache entry should be empty");
+ index = trim_index(index + 1);
+ }
+
+ guarantee(index == _first, "we should have reached where we started from");
+ return true;
+}
+#endif // PRODUCT
+
+void CSetChooserCache::insert(HeapRegion *hr) {
+ assert(!is_full(), "cache should not be empty");
+ hr->calc_gc_efficiency();
+
+ int empty_index;
+ if (_occupancy == 0) {
+ empty_index = _first;
+ } else {
+ empty_index = trim_index(_first + _occupancy);
+ assert(_cache[empty_index] == NULL, "last slot should be empty");
+ int last_index = trim_index(empty_index - 1);
+ HeapRegion *last = _cache[last_index];
+ assert(last != NULL,"as the cache is not empty, last should not be empty");
+ while (empty_index != _first &&
+ last->gc_efficiency() < hr->gc_efficiency()) {
+ _cache[empty_index] = last;
+ last->set_sort_index(get_sort_index(empty_index));
+ empty_index = last_index;
+ last_index = trim_index(last_index - 1);
+ last = _cache[last_index];
+ }
+ }
+ _cache[empty_index] = hr;
+ hr->set_sort_index(get_sort_index(empty_index));
+
+ ++_occupancy;
+ assert(verify(), "cache should be consistent");
+}
+
+HeapRegion *CSetChooserCache::remove_first() {
+ if (_occupancy > 0) {
+ assert(_cache[_first] != NULL, "cache should have at least one region");
+ HeapRegion *ret = _cache[_first];
+ _cache[_first] = NULL;
+ ret->set_sort_index(-1);
+ --_occupancy;
+ _first = trim_index(_first + 1);
+ assert(verify(), "cache should be consistent");
+ return ret;
+ } else {
+ return NULL;
+ }
+}
+
+// this is a bit expensive... but we expect that it should not be called
+// to often.
+void CSetChooserCache::remove(HeapRegion *hr) {
+ assert(_occupancy > 0, "cache should not be empty");
+ assert(hr->sort_index() < -1, "should already be in the cache");
+ int index = get_index(hr->sort_index());
+ assert(_cache[index] == hr, "index should be correct");
+ int next_index = trim_index(index + 1);
+ int last_index = trim_index(_first + _occupancy - 1);
+ while (index != last_index) {
+ assert(_cache[next_index] != NULL, "should not be null");
+ _cache[index] = _cache[next_index];
+ _cache[index]->set_sort_index(get_sort_index(index));
+
+ index = next_index;
+ next_index = trim_index(next_index+1);
+ }
+ assert(index == last_index, "should have reached the last one");
+ _cache[index] = NULL;
+ hr->set_sort_index(-1);
+ --_occupancy;
+ assert(verify(), "cache should be consistent");
+}
+
+static inline int orderRegions(HeapRegion* hr1, HeapRegion* hr2) {
+ if (hr1 == NULL) {
+ if (hr2 == NULL) return 0;
+ else return 1;
+ } else if (hr2 == NULL) {
+ return -1;
+ }
+ if (hr2->gc_efficiency() < hr1->gc_efficiency()) return -1;
+ else if (hr1->gc_efficiency() < hr2->gc_efficiency()) return 1;
+ else return 0;
+}
+
+static int orderRegions(HeapRegion** hr1p, HeapRegion** hr2p) {
+ return orderRegions(*hr1p, *hr2p);
+}
+
+CollectionSetChooser::CollectionSetChooser() :
+ // The line below is the worst bit of C++ hackery I've ever written
+ // (Detlefs, 11/23). You should think of it as equivalent to
+ // "_regions(100, true)": initialize the growable array and inform it
+ // that it should allocate its elem array(s) on the C heap. The first
+ // argument, however, is actually a comma expression (new-expr, 100).
+ // The purpose of the new_expr is to inform the growable array that it
+ // is *already* allocated on the C heap: it uses the placement syntax to
+ // keep it from actually doing any allocation.
+ _markedRegions((ResourceObj::operator new (sizeof(GrowableArray),
+ (void*)&_markedRegions,
+ ResourceObj::C_HEAP),
+ 100),
+ true),
+ _curMarkedIndex(0),
+ _numMarkedRegions(0),
+ _unmarked_age_1_returned_as_new(false),
+ _first_par_unreserved_idx(0)
+{}
+
+
+
+#ifndef PRODUCT
+bool CollectionSetChooser::verify() {
+ int index = 0;
+ guarantee(_curMarkedIndex <= _numMarkedRegions,
+ "_curMarkedIndex should be within bounds");
+ while (index < _curMarkedIndex) {
+ guarantee(_markedRegions.at(index++) == NULL,
+ "all entries before _curMarkedIndex should be NULL");
+ }
+ HeapRegion *prev = NULL;
+ while (index < _numMarkedRegions) {
+ HeapRegion *curr = _markedRegions.at(index++);
+ if (curr != NULL) {
+ int si = curr->sort_index();
+ guarantee(!curr->is_young(), "should not be young!");
+ guarantee(si > -1 && si == (index-1), "sort index invariant");
+ if (prev != NULL) {
+ guarantee(orderRegions(prev, curr) != 1, "regions should be sorted");
+ }
+ prev = curr;
+ }
+ }
+ return _cache.verify();
+}
+#endif
+
+bool
+CollectionSetChooser::addRegionToCache() {
+ assert(!_cache.is_full(), "cache should not be full");
+
+ HeapRegion *hr = NULL;
+ while (hr == NULL && _curMarkedIndex < _numMarkedRegions) {
+ hr = _markedRegions.at(_curMarkedIndex++);
+ }
+ if (hr == NULL)
+ return false;
+ assert(!hr->is_young(), "should not be young!");
+ assert(hr->sort_index() == _curMarkedIndex-1, "sort_index invariant");
+ _markedRegions.at_put(hr->sort_index(), NULL);
+ _cache.insert(hr);
+ assert(!_cache.is_empty(), "cache should not be empty");
+ assert(verify(), "cache should be consistent");
+ return false;
+}
+
+void
+CollectionSetChooser::fillCache() {
+ while (!_cache.is_full() && addRegionToCache()) {
+ }
+}
+
+void
+CollectionSetChooser::sortMarkedHeapRegions() {
+ guarantee(_cache.is_empty(), "cache should be empty");
+ // First trim any unused portion of the top in the parallel case.
+ if (_first_par_unreserved_idx > 0) {
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print(" Truncating _markedRegions from %d to %d.\n",
+ _markedRegions.length(), _first_par_unreserved_idx);
+ }
+ assert(_first_par_unreserved_idx <= _markedRegions.length(),
+ "Or we didn't reserved enough length");
+ _markedRegions.trunc_to(_first_par_unreserved_idx);
+ }
+ _markedRegions.sort(orderRegions);
+ assert(_numMarkedRegions <= _markedRegions.length(), "Requirement");
+ assert(_numMarkedRegions == 0
+ || _markedRegions.at(_numMarkedRegions-1) != NULL,
+ "Testing _numMarkedRegions");
+ assert(_numMarkedRegions == _markedRegions.length()
+ || _markedRegions.at(_numMarkedRegions) == NULL,
+ "Testing _numMarkedRegions");
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print_cr(" Sorted %d marked regions.", _numMarkedRegions);
+ }
+ for (int i = 0; i < _numMarkedRegions; i++) {
+ assert(_markedRegions.at(i) != NULL, "Should be true by sorting!");
+ _markedRegions.at(i)->set_sort_index(i);
+ if (G1PrintRegionLivenessInfo > 0) {
+ if (i == 0) gclog_or_tty->print_cr("Sorted marked regions:");
+ if (i < G1PrintRegionLivenessInfo ||
+ (_numMarkedRegions-i) < G1PrintRegionLivenessInfo) {
+ HeapRegion* hr = _markedRegions.at(i);
+ size_t u = hr->used();
+ gclog_or_tty->print_cr(" Region %d: %d used, %d max live, %5.2f%%.",
+ i, u, hr->max_live_bytes(),
+ 100.0*(float)hr->max_live_bytes()/(float)u);
+ }
+ }
+ }
+ if (G1PolicyVerbose > 1)
+ printSortedHeapRegions();
+ assert(verify(), "should now be sorted");
+}
+
+void
+printHeapRegion(HeapRegion *hr) {
+ if (hr->isHumongous())
+ gclog_or_tty->print("H: ");
+ if (hr->in_collection_set())
+ gclog_or_tty->print("CS: ");
+ if (hr->popular())
+ gclog_or_tty->print("pop: ");
+ gclog_or_tty->print_cr("Region " PTR_FORMAT " (%s%s) "
+ "[" PTR_FORMAT ", " PTR_FORMAT"] "
+ "Used: " SIZE_FORMAT "K, garbage: " SIZE_FORMAT "K.",
+ hr, hr->is_young() ? "Y " : " ",
+ hr->is_marked()? "M1" : "M0",
+ hr->bottom(), hr->end(),
+ hr->used()/K, hr->garbage_bytes()/K);
+}
+
+void
+CollectionSetChooser::addMarkedHeapRegion(HeapRegion* hr) {
+ assert(!hr->isHumongous(),
+ "Humongous regions shouldn't be added to the collection set");
+ assert(!hr->is_young(), "should not be young!");
+ _markedRegions.append(hr);
+ _numMarkedRegions++;
+ hr->calc_gc_efficiency();
+}
+
+void
+CollectionSetChooser::
+prepareForAddMarkedHeapRegionsPar(size_t n_regions, size_t chunkSize) {
+ _first_par_unreserved_idx = 0;
+ size_t max_waste = ParallelGCThreads * chunkSize;
+ // it should be aligned with respect to chunkSize
+ size_t aligned_n_regions =
+ (n_regions + (chunkSize - 1)) / chunkSize * chunkSize;
+ assert( aligned_n_regions % chunkSize == 0, "should be aligned" );
+ _markedRegions.at_put_grow((int)(aligned_n_regions + max_waste - 1), NULL);
+}
+
+jint
+CollectionSetChooser::getParMarkedHeapRegionChunk(jint n_regions) {
+ jint res = Atomic::add(n_regions, &_first_par_unreserved_idx);
+ assert(_markedRegions.length() > res + n_regions - 1,
+ "Should already have been expanded");
+ return res - n_regions;
+}
+
+void
+CollectionSetChooser::setMarkedHeapRegion(jint index, HeapRegion* hr) {
+ assert(_markedRegions.at(index) == NULL, "precondition");
+ assert(!hr->is_young(), "should not be young!");
+ _markedRegions.at_put(index, hr);
+ hr->calc_gc_efficiency();
+}
+
+void
+CollectionSetChooser::incNumMarkedHeapRegions(jint inc_by) {
+ (void)Atomic::add(inc_by, &_numMarkedRegions);
+}
+
+void
+CollectionSetChooser::clearMarkedHeapRegions(){
+ for (int i = 0; i < _markedRegions.length(); i++) {
+ HeapRegion* r = _markedRegions.at(i);
+ if (r != NULL) r->set_sort_index(-1);
+ }
+ _markedRegions.clear();
+ _curMarkedIndex = 0;
+ _numMarkedRegions = 0;
+ _cache.clear();
+};
+
+void
+CollectionSetChooser::updateAfterFullCollection() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ clearMarkedHeapRegions();
+}
+
+void
+CollectionSetChooser::printSortedHeapRegions() {
+ gclog_or_tty->print_cr("Printing %d Heap Regions sorted by amount of known garbage",
+ _numMarkedRegions);
+ for (int i = 0; i < _markedRegions.length(); i++) {
+ printHeapRegion(_markedRegions.at(i));
+ }
+ gclog_or_tty->print_cr("Done sorted heap region print");
+}
+
+void CollectionSetChooser::removeRegion(HeapRegion *hr) {
+ int si = hr->sort_index();
+ assert(si == -1 || hr->is_marked(), "Sort index not valid.");
+ if (si > -1) {
+ assert(_markedRegions.at(si) == hr, "Sort index not valid." );
+ _markedRegions.at_put(si, NULL);
+ } else if (si < -1) {
+ assert(_cache.region_in_cache(hr), "should be in the cache");
+ _cache.remove(hr);
+ assert(hr->sort_index() == -1, "sort index invariant");
+ }
+ hr->set_sort_index(-1);
+}
+
+// if time_remaining < 0.0, then this method should try to return
+// a region, whether it fits within the remaining time or not
+HeapRegion*
+CollectionSetChooser::getNextMarkedRegion(double time_remaining,
+ double avg_prediction) {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ G1CollectorPolicy* g1p = g1h->g1_policy();
+ fillCache();
+ if (_cache.is_empty()) {
+ assert(_curMarkedIndex == _numMarkedRegions,
+ "if cache is empty, list should also be empty");
+ return NULL;
+ }
+
+ HeapRegion *hr = _cache.get_first();
+ assert(hr != NULL, "if cache not empty, first entry should be non-null");
+ double predicted_time = g1h->predict_region_elapsed_time_ms(hr, false);
+
+ if (g1p->adaptive_young_list_length()) {
+ if (time_remaining - predicted_time < 0.0) {
+ g1h->check_if_region_is_too_expensive(predicted_time);
+ return NULL;
+ }
+ } else {
+ if (predicted_time > 2.0 * avg_prediction) {
+ return NULL;
+ }
+ }
+
+ HeapRegion *hr2 = _cache.remove_first();
+ assert(hr == hr2, "cache contents should not have changed");
+
+ return hr;
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp b/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp
new file mode 100644
index 00000000000..60d8bf2057e
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/collectionSetChooser.hpp
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// We need to sort heap regions by collection desirability.
+
+class CSetChooserCache {
+private:
+ enum {
+ CacheLength = 16
+ } PrivateConstants;
+
+ HeapRegion* _cache[CacheLength];
+ int _occupancy; // number of region in cache
+ int _first; // "first" region in the cache
+
+ // adding CacheLength to deal with negative values
+ inline int trim_index(int index) {
+ return (index + CacheLength) % CacheLength;
+ }
+
+ inline int get_sort_index(int index) {
+ return -index-2;
+ }
+ inline int get_index(int sort_index) {
+ return -sort_index-2;
+ }
+
+public:
+ CSetChooserCache(void);
+
+ inline int occupancy(void) { return _occupancy; }
+ inline bool is_full() { return _occupancy == CacheLength; }
+ inline bool is_empty() { return _occupancy == 0; }
+
+ void clear(void);
+ void insert(HeapRegion *hr);
+ HeapRegion *remove_first(void);
+ void remove (HeapRegion *hr);
+ inline HeapRegion *get_first(void) {
+ return _cache[_first];
+ }
+
+#ifndef PRODUCT
+ bool verify (void);
+ bool region_in_cache(HeapRegion *hr) {
+ int sort_index = hr->sort_index();
+ if (sort_index < -1) {
+ int index = get_index(sort_index);
+ guarantee(index < CacheLength, "should be within bounds");
+ return _cache[index] == hr;
+ } else
+ return 0;
+ }
+#endif // PRODUCT
+};
+
+class CollectionSetChooser: public CHeapObj {
+
+ GrowableArray _markedRegions;
+ int _curMarkedIndex;
+ int _numMarkedRegions;
+ CSetChooserCache _cache;
+
+ // True iff last collection pause ran of out new "age 0" regions, and
+ // returned an "age 1" region.
+ bool _unmarked_age_1_returned_as_new;
+
+ jint _first_par_unreserved_idx;
+
+public:
+
+ HeapRegion* getNextMarkedRegion(double time_so_far, double avg_prediction);
+
+ CollectionSetChooser();
+
+ void printSortedHeapRegions();
+
+ void sortMarkedHeapRegions();
+ void fillCache();
+ bool addRegionToCache(void);
+ void addMarkedHeapRegion(HeapRegion *hr);
+
+ // Must be called before calls to getParMarkedHeapRegionChunk.
+ // "n_regions" is the number of regions, "chunkSize" the chunk size.
+ void prepareForAddMarkedHeapRegionsPar(size_t n_regions, size_t chunkSize);
+ // Returns the first index in a contiguous chunk of "n_regions" indexes
+ // that the calling thread has reserved. These must be set by the
+ // calling thread using "setMarkedHeapRegion" (to NULL if necessary).
+ jint getParMarkedHeapRegionChunk(jint n_regions);
+ // Set the marked array entry at index to hr. Careful to claim the index
+ // first if in parallel.
+ void setMarkedHeapRegion(jint index, HeapRegion* hr);
+ // Atomically increment the number of claimed regions by "inc_by".
+ void incNumMarkedHeapRegions(jint inc_by);
+
+ void clearMarkedHeapRegions();
+
+ void updateAfterFullCollection();
+
+ // Ensure that "hr" is not a member of the marked region array or the cache
+ void removeRegion(HeapRegion* hr);
+
+ bool unmarked_age_1_returned_as_new() { return _unmarked_age_1_returned_as_new; }
+
+ // Returns true if the used portion of "_markedRegions" is properly
+ // sorted, otherwise asserts false.
+#ifndef PRODUCT
+ bool verify(void);
+ bool regionProperlyOrdered(HeapRegion* r) {
+ int si = r->sort_index();
+ return (si == -1) ||
+ (si > -1 && _markedRegions.at(si) == r) ||
+ (si < -1 && _cache.region_in_cache(r));
+ }
+#endif
+
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp
new file mode 100644
index 00000000000..2eb2bc0ca69
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_concurrentG1Refine.cpp.incl"
+
+bool ConcurrentG1Refine::_enabled = false;
+
+ConcurrentG1Refine::ConcurrentG1Refine() :
+ _pya(PYA_continue), _last_pya(PYA_continue),
+ _last_cards_during(), _first_traversal(false),
+ _card_counts(NULL), _cur_card_count_histo(NULL), _cum_card_count_histo(NULL),
+ _hot_cache(NULL),
+ _def_use_cache(false), _use_cache(false),
+ _n_periods(0), _total_cards(0), _total_travs(0)
+{
+ if (G1ConcRefine) {
+ _cg1rThread = new ConcurrentG1RefineThread(this);
+ assert(cg1rThread() != NULL, "Conc refine should have been created");
+ assert(cg1rThread()->cg1r() == this,
+ "Conc refine thread should refer to this");
+ } else {
+ _cg1rThread = NULL;
+ }
+}
+
+void ConcurrentG1Refine::init() {
+ if (G1ConcRSLogCacheSize > 0 || G1ConcRSCountTraversals) {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ _n_card_counts =
+ (unsigned) (g1h->g1_reserved_obj_bytes() >> CardTableModRefBS::card_shift);
+ _card_counts = NEW_C_HEAP_ARRAY(unsigned char, _n_card_counts);
+ for (size_t i = 0; i < _n_card_counts; i++) _card_counts[i] = 0;
+ ModRefBarrierSet* bs = g1h->mr_bs();
+ guarantee(bs->is_a(BarrierSet::CardTableModRef), "Precondition");
+ CardTableModRefBS* ctbs = (CardTableModRefBS*)bs;
+ _ct_bot = ctbs->byte_for_const(g1h->reserved_region().start());
+ if (G1ConcRSCountTraversals) {
+ _cur_card_count_histo = NEW_C_HEAP_ARRAY(unsigned, 256);
+ _cum_card_count_histo = NEW_C_HEAP_ARRAY(unsigned, 256);
+ for (int i = 0; i < 256; i++) {
+ _cur_card_count_histo[i] = 0;
+ _cum_card_count_histo[i] = 0;
+ }
+ }
+ }
+ if (G1ConcRSLogCacheSize > 0) {
+ _def_use_cache = true;
+ _use_cache = true;
+ _hot_cache_size = (1 << G1ConcRSLogCacheSize);
+ _hot_cache = NEW_C_HEAP_ARRAY(jbyte*, _hot_cache_size);
+ _n_hot = 0;
+ _hot_cache_idx = 0;
+ }
+}
+
+ConcurrentG1Refine::~ConcurrentG1Refine() {
+ if (G1ConcRSLogCacheSize > 0 || G1ConcRSCountTraversals) {
+ assert(_card_counts != NULL, "Logic");
+ FREE_C_HEAP_ARRAY(unsigned char, _card_counts);
+ assert(_cur_card_count_histo != NULL, "Logic");
+ FREE_C_HEAP_ARRAY(unsigned, _cur_card_count_histo);
+ assert(_cum_card_count_histo != NULL, "Logic");
+ FREE_C_HEAP_ARRAY(unsigned, _cum_card_count_histo);
+ }
+ if (G1ConcRSLogCacheSize > 0) {
+ assert(_hot_cache != NULL, "Logic");
+ FREE_C_HEAP_ARRAY(jbyte*, _hot_cache);
+ }
+}
+
+bool ConcurrentG1Refine::refine() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ unsigned cards_before = g1h->g1_rem_set()->conc_refine_cards();
+ clear_hot_cache(); // Any previous values in this are now invalid.
+ g1h->g1_rem_set()->concurrentRefinementPass(this);
+ _traversals++;
+ unsigned cards_after = g1h->g1_rem_set()->conc_refine_cards();
+ unsigned cards_during = cards_after-cards_before;
+ // If this is the first traversal in the current enabling
+ // and we did some cards, or if the number of cards found is decreasing
+ // sufficiently quickly, then keep going. Otherwise, sleep a while.
+ bool res =
+ (_first_traversal && cards_during > 0)
+ ||
+ (!_first_traversal && cards_during * 3 < _last_cards_during * 2);
+ _last_cards_during = cards_during;
+ _first_traversal = false;
+ return res;
+}
+
+void ConcurrentG1Refine::enable() {
+ MutexLocker x(G1ConcRefine_mon);
+ if (!_enabled) {
+ _enabled = true;
+ _first_traversal = true; _last_cards_during = 0;
+ G1ConcRefine_mon->notify_all();
+ }
+}
+
+unsigned ConcurrentG1Refine::disable() {
+ MutexLocker x(G1ConcRefine_mon);
+ if (_enabled) {
+ _enabled = false;
+ return _traversals;
+ } else {
+ return 0;
+ }
+}
+
+void ConcurrentG1Refine::wait_for_ConcurrentG1Refine_enabled() {
+ G1ConcRefine_mon->lock();
+ while (!_enabled) {
+ G1ConcRefine_mon->wait(Mutex::_no_safepoint_check_flag);
+ }
+ G1ConcRefine_mon->unlock();
+ _traversals = 0;
+};
+
+void ConcurrentG1Refine::set_pya_restart() {
+ // If we're using the log-based RS barrier, the above will cause
+ // in-progress traversals of completed log buffers to quit early; we will
+ // also abandon all other buffers.
+ if (G1RSBarrierUseQueue) {
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ dcqs.abandon_logs();
+ if (_cg1rThread->do_traversal()) {
+ _pya = PYA_restart;
+ } else {
+ _cg1rThread->set_do_traversal(true);
+ // Reset the post-yield actions.
+ _pya = PYA_continue;
+ _last_pya = PYA_continue;
+ }
+ } else {
+ _pya = PYA_restart;
+ }
+}
+
+void ConcurrentG1Refine::set_pya_cancel() {
+ _pya = PYA_cancel;
+}
+
+PostYieldAction ConcurrentG1Refine::get_pya() {
+ if (_pya != PYA_continue) {
+ jint val = _pya;
+ while (true) {
+ jint val_read = Atomic::cmpxchg(PYA_continue, &_pya, val);
+ if (val_read == val) {
+ PostYieldAction res = (PostYieldAction)val;
+ assert(res != PYA_continue, "Only the refine thread should reset.");
+ _last_pya = res;
+ return res;
+ } else {
+ val = val_read;
+ }
+ }
+ }
+ // QQQ WELL WHAT DO WE RETURN HERE???
+ // make up something!
+ return PYA_continue;
+}
+
+PostYieldAction ConcurrentG1Refine::get_last_pya() {
+ PostYieldAction res = _last_pya;
+ _last_pya = PYA_continue;
+ return res;
+}
+
+bool ConcurrentG1Refine::do_traversal() {
+ return _cg1rThread->do_traversal();
+}
+
+int ConcurrentG1Refine::add_card_count(jbyte* card_ptr) {
+ size_t card_num = (card_ptr - _ct_bot);
+ guarantee(0 <= card_num && card_num < _n_card_counts, "Bounds");
+ unsigned char cnt = _card_counts[card_num];
+ if (cnt < 255) _card_counts[card_num]++;
+ return cnt;
+ _total_travs++;
+}
+
+jbyte* ConcurrentG1Refine::cache_insert(jbyte* card_ptr) {
+ int count = add_card_count(card_ptr);
+ // Count previously unvisited cards.
+ if (count == 0) _total_cards++;
+ // We'll assume a traversal unless we store it in the cache.
+ if (count < G1ConcRSHotCardLimit) {
+ _total_travs++;
+ return card_ptr;
+ }
+ // Otherwise, it's hot.
+ jbyte* res = NULL;
+ MutexLockerEx x(HotCardCache_lock, Mutex::_no_safepoint_check_flag);
+ if (_n_hot == _hot_cache_size) {
+ _total_travs++;
+ res = _hot_cache[_hot_cache_idx];
+ _n_hot--;
+ }
+ // Now _n_hot < _hot_cache_size, and we can insert at _hot_cache_idx.
+ _hot_cache[_hot_cache_idx] = card_ptr;
+ _hot_cache_idx++;
+ if (_hot_cache_idx == _hot_cache_size) _hot_cache_idx = 0;
+ _n_hot++;
+ return res;
+}
+
+
+void ConcurrentG1Refine::clean_up_cache(int worker_i, G1RemSet* g1rs) {
+ assert(!use_cache(), "cache should be disabled");
+ int start_ind = _hot_cache_idx-1;
+ for (int i = 0; i < _n_hot; i++) {
+ int ind = start_ind - i;
+ if (ind < 0) ind = ind + _hot_cache_size;
+ jbyte* entry = _hot_cache[ind];
+ if (entry != NULL) {
+ g1rs->concurrentRefineOneCard(entry, worker_i);
+ }
+ }
+ _n_hot = 0;
+ _hot_cache_idx = 0;
+}
+
+void ConcurrentG1Refine::clear_and_record_card_counts() {
+ if (G1ConcRSLogCacheSize == 0 && !G1ConcRSCountTraversals) return;
+ _n_periods++;
+ if (G1ConcRSCountTraversals) {
+ for (size_t i = 0; i < _n_card_counts; i++) {
+ unsigned char bucket = _card_counts[i];
+ _cur_card_count_histo[bucket]++;
+ _card_counts[i] = 0;
+ }
+ gclog_or_tty->print_cr("Card counts:");
+ for (int i = 0; i < 256; i++) {
+ if (_cur_card_count_histo[i] > 0) {
+ gclog_or_tty->print_cr(" %3d: %9d", i, _cur_card_count_histo[i]);
+ _cum_card_count_histo[i] += _cur_card_count_histo[i];
+ _cur_card_count_histo[i] = 0;
+ }
+ }
+ } else {
+ assert(G1ConcRSLogCacheSize > 0, "Logic");
+ Copy::fill_to_words((HeapWord*)(&_card_counts[0]),
+ _n_card_counts / HeapWordSize);
+ }
+}
+
+void
+ConcurrentG1Refine::
+print_card_count_histo_range(unsigned* histo, int from, int to,
+ float& cum_card_pct,
+ float& cum_travs_pct) {
+ unsigned cards = 0;
+ unsigned travs = 0;
+ guarantee(to <= 256, "Precondition");
+ for (int i = from; i < to-1; i++) {
+ cards += histo[i];
+ travs += histo[i] * i;
+ }
+ if (to == 256) {
+ unsigned histo_card_sum = 0;
+ unsigned histo_trav_sum = 0;
+ for (int i = 1; i < 255; i++) {
+ histo_trav_sum += histo[i] * i;
+ }
+ cards += histo[255];
+ // correct traversals for the last one.
+ unsigned travs_255 = (unsigned) (_total_travs - histo_trav_sum);
+ travs += travs_255;
+
+ } else {
+ cards += histo[to-1];
+ travs += histo[to-1] * (to-1);
+ }
+ float fperiods = (float)_n_periods;
+ float f_tot_cards = (float)_total_cards/fperiods;
+ float f_tot_travs = (float)_total_travs/fperiods;
+ if (cards > 0) {
+ float fcards = (float)cards/fperiods;
+ float ftravs = (float)travs/fperiods;
+ if (to == 256) {
+ gclog_or_tty->print(" %4d- %10.2f%10.2f", from, fcards, ftravs);
+ } else {
+ gclog_or_tty->print(" %4d-%4d %10.2f%10.2f", from, to-1, fcards, ftravs);
+ }
+ float pct_cards = fcards*100.0/f_tot_cards;
+ cum_card_pct += pct_cards;
+ float pct_travs = ftravs*100.0/f_tot_travs;
+ cum_travs_pct += pct_travs;
+ gclog_or_tty->print_cr("%10.2f%10.2f%10.2f%10.2f",
+ pct_cards, cum_card_pct,
+ pct_travs, cum_travs_pct);
+ }
+}
+
+void ConcurrentG1Refine::print_final_card_counts() {
+ if (!G1ConcRSCountTraversals) return;
+
+ gclog_or_tty->print_cr("Did %d total traversals of %d distinct cards.",
+ _total_travs, _total_cards);
+ float fperiods = (float)_n_periods;
+ gclog_or_tty->print_cr(" This is an average of %8.2f traversals, %8.2f cards, "
+ "per collection.", (float)_total_travs/fperiods,
+ (float)_total_cards/fperiods);
+ gclog_or_tty->print_cr(" This is an average of %8.2f traversals/distinct "
+ "dirty card.\n",
+ _total_cards > 0 ?
+ (float)_total_travs/(float)_total_cards : 0.0);
+
+
+ gclog_or_tty->print_cr("Histogram:\n\n%10s %10s%10s%10s%10s%10s%10s",
+ "range", "# cards", "# travs", "% cards", "(cum)",
+ "% travs", "(cum)");
+ gclog_or_tty->print_cr("------------------------------------------------------------"
+ "-------------");
+ float cum_cards_pct = 0.0;
+ float cum_travs_pct = 0.0;
+ for (int i = 1; i < 10; i++) {
+ print_card_count_histo_range(_cum_card_count_histo, i, i+1,
+ cum_cards_pct, cum_travs_pct);
+ }
+ for (int i = 10; i < 100; i += 10) {
+ print_card_count_histo_range(_cum_card_count_histo, i, i+10,
+ cum_cards_pct, cum_travs_pct);
+ }
+ print_card_count_histo_range(_cum_card_count_histo, 100, 150,
+ cum_cards_pct, cum_travs_pct);
+ print_card_count_histo_range(_cum_card_count_histo, 150, 200,
+ cum_cards_pct, cum_travs_pct);
+ print_card_count_histo_range(_cum_card_count_histo, 150, 255,
+ cum_cards_pct, cum_travs_pct);
+ print_card_count_histo_range(_cum_card_count_histo, 255, 256,
+ cum_cards_pct, cum_travs_pct);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp
new file mode 100644
index 00000000000..ea9a997d7a6
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// Forward decl
+class ConcurrentG1RefineThread;
+class G1RemSet;
+
+// What to do after a yield:
+enum PostYieldAction {
+ PYA_continue, // Continue the traversal
+ PYA_restart, // Restart
+ PYA_cancel // It's been completed by somebody else: cancel.
+};
+
+class ConcurrentG1Refine {
+ ConcurrentG1RefineThread* _cg1rThread;
+
+ volatile jint _pya;
+ PostYieldAction _last_pya;
+
+ static bool _enabled; // Protected by G1ConcRefine_mon.
+ unsigned _traversals;
+
+ // Number of cards processed during last refinement traversal.
+ unsigned _first_traversal;
+ unsigned _last_cards_during;
+
+ // The cache for card refinement.
+ bool _use_cache;
+ bool _def_use_cache;
+ size_t _n_periods;
+ size_t _total_cards;
+ size_t _total_travs;
+
+ unsigned char* _card_counts;
+ unsigned _n_card_counts;
+ const jbyte* _ct_bot;
+ unsigned* _cur_card_count_histo;
+ unsigned* _cum_card_count_histo;
+ jbyte** _hot_cache;
+ int _hot_cache_size;
+ int _n_hot;
+ int _hot_cache_idx;
+
+ // Returns the count of this card after incrementing it.
+ int add_card_count(jbyte* card_ptr);
+
+ void print_card_count_histo_range(unsigned* histo, int from, int to,
+ float& cum_card_pct,
+ float& cum_travs_pct);
+ public:
+ ConcurrentG1Refine();
+ ~ConcurrentG1Refine();
+
+ void init(); // Accomplish some initialization that has to wait.
+
+ // Enabled Conc refinement, waking up thread if necessary.
+ void enable();
+
+ // Returns the number of traversals performed since this refiner was enabled.
+ unsigned disable();
+
+ // Requires G1ConcRefine_mon to be held.
+ bool enabled() { return _enabled; }
+
+ // Returns only when G1 concurrent refinement has been enabled.
+ void wait_for_ConcurrentG1Refine_enabled();
+
+ // Do one concurrent refinement pass over the card table. Returns "true"
+ // if heuristics determine that another pass should be done immediately.
+ bool refine();
+
+ // Indicate that an in-progress refinement pass should start over.
+ void set_pya_restart();
+ // Indicate that an in-progress refinement pass should quit.
+ void set_pya_cancel();
+
+ // Get the appropriate post-yield action. Also sets last_pya.
+ PostYieldAction get_pya();
+
+ // The last PYA read by "get_pya".
+ PostYieldAction get_last_pya();
+
+ bool do_traversal();
+
+ ConcurrentG1RefineThread* cg1rThread() { return _cg1rThread; }
+
+ // If this is the first entry for the slot, writes into the cache and
+ // returns NULL. If it causes an eviction, returns the evicted pointer.
+ // Otherwise, its a cache hit, and returns NULL.
+ jbyte* cache_insert(jbyte* card_ptr);
+
+ // Process the cached entries.
+ void clean_up_cache(int worker_i, G1RemSet* g1rs);
+
+ // Discard entries in the hot cache.
+ void clear_hot_cache() {
+ _hot_cache_idx = 0; _n_hot = 0;
+ }
+
+ bool hot_cache_is_empty() { return _n_hot == 0; }
+
+ bool use_cache() { return _use_cache; }
+ void set_use_cache(bool b) {
+ if (b) _use_cache = _def_use_cache;
+ else _use_cache = false;
+ }
+
+ void clear_and_record_card_counts();
+ void print_final_card_counts();
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp
new file mode 100644
index 00000000000..110c08327c3
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp
@@ -0,0 +1,246 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_concurrentG1RefineThread.cpp.incl"
+
+// ======= Concurrent Mark Thread ========
+
+// The CM thread is created when the G1 garbage collector is used
+
+ConcurrentG1RefineThread::
+ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r) :
+ ConcurrentGCThread(),
+ _cg1r(cg1r),
+ _started(false),
+ _in_progress(false),
+ _do_traversal(false),
+ _vtime_accum(0.0),
+ _co_tracker(G1CRGroup),
+ _interval_ms(5.0)
+{
+ create_and_start();
+}
+
+const long timeout = 200; // ms.
+
+void ConcurrentG1RefineThread::traversalBasedRefinement() {
+ _cg1r->wait_for_ConcurrentG1Refine_enabled();
+ MutexLocker x(G1ConcRefine_mon);
+ while (_cg1r->enabled()) {
+ MutexUnlocker ux(G1ConcRefine_mon);
+ ResourceMark rm;
+ HandleMark hm;
+
+ if (TraceG1Refine) gclog_or_tty->print_cr("G1-Refine starting pass");
+ _sts.join();
+ bool no_sleep = _cg1r->refine();
+ _sts.leave();
+ if (!no_sleep) {
+ MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+ // We do this only for the timeout; we don't expect this to be signalled.
+ CGC_lock->wait(Mutex::_no_safepoint_check_flag, timeout);
+ }
+ }
+}
+
+void ConcurrentG1RefineThread::queueBasedRefinement() {
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ // Wait for completed log buffers to exist.
+ {
+ MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag);
+ while (!_do_traversal && !dcqs.process_completed_buffers() &&
+ !_should_terminate) {
+ DirtyCardQ_CBL_mon->wait(Mutex::_no_safepoint_check_flag);
+ }
+ }
+
+ if (_should_terminate) {
+ return;
+ }
+
+ // Now we take them off (this doesn't hold locks while it applies
+ // closures.) (If we did a full collection, then we'll do a full
+ // traversal.
+ _sts.join();
+ if (_do_traversal) {
+ (void)_cg1r->refine();
+ switch (_cg1r->get_last_pya()) {
+ case PYA_cancel: case PYA_continue:
+ // Continue was caught and handled inside "refine". If it's still
+ // "continue" when we get here, we're done.
+ _do_traversal = false;
+ break;
+ case PYA_restart:
+ assert(_do_traversal, "Because of Full GC.");
+ break;
+ }
+ } else {
+ int n_logs = 0;
+ int lower_limit = 0;
+ double start_vtime_sec; // only used when G1SmoothConcRefine is on
+ int prev_buffer_num; // only used when G1SmoothConcRefine is on
+
+ if (G1SmoothConcRefine) {
+ lower_limit = 0;
+ start_vtime_sec = os::elapsedVTime();
+ prev_buffer_num = (int) dcqs.completed_buffers_num();
+ } else {
+ lower_limit = DCQBarrierProcessCompletedThreshold / 4; // For now.
+ }
+ while (dcqs.apply_closure_to_completed_buffer(0, lower_limit)) {
+ double end_vtime_sec;
+ double elapsed_vtime_sec;
+ int elapsed_vtime_ms;
+ int curr_buffer_num;
+
+ if (G1SmoothConcRefine) {
+ end_vtime_sec = os::elapsedVTime();
+ elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
+ elapsed_vtime_ms = (int) (elapsed_vtime_sec * 1000.0);
+ curr_buffer_num = (int) dcqs.completed_buffers_num();
+
+ if (curr_buffer_num > prev_buffer_num ||
+ curr_buffer_num > DCQBarrierProcessCompletedThreshold) {
+ decreaseInterval(elapsed_vtime_ms);
+ } else if (curr_buffer_num < prev_buffer_num) {
+ increaseInterval(elapsed_vtime_ms);
+ }
+ }
+
+ sample_young_list_rs_lengths();
+ _co_tracker.update(false);
+
+ if (G1SmoothConcRefine) {
+ start_vtime_sec = os::elapsedVTime();
+ prev_buffer_num = curr_buffer_num;
+
+ _sts.leave();
+ os::sleep(Thread::current(), (jlong) _interval_ms, false);
+ _sts.join();
+ }
+
+ n_logs++;
+ }
+ // Make sure we harvest the PYA, if any.
+ (void)_cg1r->get_pya();
+ }
+ _sts.leave();
+}
+
+void ConcurrentG1RefineThread::sample_young_list_rs_lengths() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ G1CollectorPolicy* g1p = g1h->g1_policy();
+ if (g1p->adaptive_young_list_length()) {
+ int regions_visited = 0;
+
+ g1h->young_list_rs_length_sampling_init();
+ while (g1h->young_list_rs_length_sampling_more()) {
+ g1h->young_list_rs_length_sampling_next();
+ ++regions_visited;
+
+ // we try to yield every time we visit 10 regions
+ if (regions_visited == 10) {
+ if (_sts.should_yield()) {
+ _sts.yield("G1 refine");
+ // we just abandon the iteration
+ break;
+ }
+ regions_visited = 0;
+ }
+ }
+
+ g1p->check_prediction_validity();
+ }
+}
+
+void ConcurrentG1RefineThread::run() {
+ initialize_in_thread();
+ _vtime_start = os::elapsedVTime();
+ wait_for_universe_init();
+
+ _co_tracker.enable();
+ _co_tracker.start();
+
+ while (!_should_terminate) {
+ // wait until started is set.
+ if (G1RSBarrierUseQueue) {
+ queueBasedRefinement();
+ } else {
+ traversalBasedRefinement();
+ }
+ _sts.join();
+ _co_tracker.update();
+ _sts.leave();
+ if (os::supports_vtime()) {
+ _vtime_accum = (os::elapsedVTime() - _vtime_start);
+ } else {
+ _vtime_accum = 0.0;
+ }
+ }
+ _sts.join();
+ _co_tracker.update(true);
+ _sts.leave();
+ assert(_should_terminate, "just checking");
+
+ terminate();
+}
+
+
+void ConcurrentG1RefineThread::yield() {
+ if (TraceG1Refine) gclog_or_tty->print_cr("G1-Refine-yield");
+ _sts.yield("G1 refine");
+ if (TraceG1Refine) gclog_or_tty->print_cr("G1-Refine-yield-end");
+}
+
+void ConcurrentG1RefineThread::stop() {
+ // it is ok to take late safepoints here, if needed
+ {
+ MutexLockerEx mu(Terminator_lock);
+ _should_terminate = true;
+ }
+
+ {
+ MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag);
+ DirtyCardQ_CBL_mon->notify_all();
+ }
+
+ {
+ MutexLockerEx mu(Terminator_lock);
+ while (!_has_terminated) {
+ Terminator_lock->wait();
+ }
+ }
+ if (TraceG1Refine) gclog_or_tty->print_cr("G1-Refine-stop");
+}
+
+void ConcurrentG1RefineThread::print() {
+ gclog_or_tty->print("\"Concurrent G1 Refinement Thread\" ");
+ Thread::print();
+ gclog_or_tty->cr();
+}
+
+void ConcurrentG1RefineThread::set_do_traversal(bool b) {
+ _do_traversal = b;
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp
new file mode 100644
index 00000000000..69f272c7895
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// Forward Decl.
+class ConcurrentG1Refine;
+
+// The G1 Concurrent Refinement Thread (could be several in the future).
+
+class ConcurrentG1RefineThread: public ConcurrentGCThread {
+ friend class VMStructs;
+ friend class G1CollectedHeap;
+
+ double _vtime_start; // Initial virtual time.
+ double _vtime_accum; // Initial virtual time.
+
+ public:
+ virtual void run();
+
+ private:
+ ConcurrentG1Refine* _cg1r;
+ bool _started;
+ bool _in_progress;
+ volatile bool _restart;
+
+ COTracker _co_tracker;
+ double _interval_ms;
+
+ bool _do_traversal;
+
+ void decreaseInterval(int processing_time_ms) {
+ double min_interval_ms = (double) processing_time_ms;
+ _interval_ms = 0.8 * _interval_ms;
+ if (_interval_ms < min_interval_ms)
+ _interval_ms = min_interval_ms;
+ }
+ void increaseInterval(int processing_time_ms) {
+ double max_interval_ms = 9.0 * (double) processing_time_ms;
+ _interval_ms = 1.1 * _interval_ms;
+ if (max_interval_ms > 0 && _interval_ms > max_interval_ms)
+ _interval_ms = max_interval_ms;
+ }
+
+ void sleepBeforeNextCycle();
+
+ void traversalBasedRefinement();
+
+ void queueBasedRefinement();
+
+ // For use by G1CollectedHeap, which is a friend.
+ static SuspendibleThreadSet* sts() { return &_sts; }
+
+ public:
+ // Constructor
+ ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r);
+
+ // Printing
+ void print();
+
+ // Total virtual time so far.
+ double vtime_accum() { return _vtime_accum; }
+
+ ConcurrentG1Refine* cg1r() { return _cg1r; }
+
+
+ void set_started() { _started = true; }
+ void clear_started() { _started = false; }
+ bool started() { return _started; }
+
+ void set_in_progress() { _in_progress = true; }
+ void clear_in_progress() { _in_progress = false; }
+ bool in_progress() { return _in_progress; }
+
+ void set_do_traversal(bool b);
+ bool do_traversal() { return _do_traversal; }
+
+ void sample_young_list_rs_lengths();
+
+ // Yield for GC
+ void yield();
+
+ // shutdown
+ static void stop();
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp
new file mode 100644
index 00000000000..c7a7ca7dc8f
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp
@@ -0,0 +1,3969 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_concurrentMark.cpp.incl"
+
+//
+// CMS Bit Map Wrapper
+
+CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter):
+ _bm((uintptr_t*)NULL,0),
+ _shifter(shifter) {
+ _bmStartWord = (HeapWord*)(rs.base());
+ _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes
+ ReservedSpace brs(ReservedSpace::allocation_align_size_up(
+ (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1));
+
+ guarantee(brs.is_reserved(), "couldn't allocate CMS bit map");
+ // For now we'll just commit all of the bit map up fromt.
+ // Later on we'll try to be more parsimonious with swap.
+ guarantee(_virtual_space.initialize(brs, brs.size()),
+ "couldn't reseve backing store for CMS bit map");
+ assert(_virtual_space.committed_size() == brs.size(),
+ "didn't reserve backing store for all of CMS bit map?");
+ _bm.set_map((uintptr_t*)_virtual_space.low());
+ assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >=
+ _bmWordSize, "inconsistency in bit map sizing");
+ _bm.set_size(_bmWordSize >> _shifter);
+}
+
+HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
+ HeapWord* limit) const {
+ // First we must round addr *up* to a possible object boundary.
+ addr = (HeapWord*)align_size_up((intptr_t)addr,
+ HeapWordSize << _shifter);
+ size_t addrOffset = heapWordToOffset(addr);
+ if (limit == NULL) limit = _bmStartWord + _bmWordSize;
+ size_t limitOffset = heapWordToOffset(limit);
+ size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset);
+ HeapWord* nextAddr = offsetToHeapWord(nextOffset);
+ assert(nextAddr >= addr, "get_next_one postcondition");
+ assert(nextAddr == limit || isMarked(nextAddr),
+ "get_next_one postcondition");
+ return nextAddr;
+}
+
+HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
+ HeapWord* limit) const {
+ size_t addrOffset = heapWordToOffset(addr);
+ if (limit == NULL) limit = _bmStartWord + _bmWordSize;
+ size_t limitOffset = heapWordToOffset(limit);
+ size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset);
+ HeapWord* nextAddr = offsetToHeapWord(nextOffset);
+ assert(nextAddr >= addr, "get_next_one postcondition");
+ assert(nextAddr == limit || !isMarked(nextAddr),
+ "get_next_one postcondition");
+ return nextAddr;
+}
+
+int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const {
+ assert((diff & ((1 << _shifter) - 1)) == 0, "argument check");
+ return (int) (diff >> _shifter);
+}
+
+bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) {
+ HeapWord* left = MAX2(_bmStartWord, mr.start());
+ HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end());
+ if (right > left) {
+ // Right-open interval [leftOffset, rightOffset).
+ return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right));
+ } else {
+ return true;
+ }
+}
+
+void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap,
+ size_t from_start_index,
+ HeapWord* to_start_word,
+ size_t word_num) {
+ _bm.mostly_disjoint_range_union(from_bitmap,
+ from_start_index,
+ heapWordToOffset(to_start_word),
+ word_num);
+}
+
+#ifndef PRODUCT
+bool CMBitMapRO::covers(ReservedSpace rs) const {
+ // assert(_bm.map() == _virtual_space.low(), "map inconsistency");
+ assert(((size_t)_bm.size() * (1 << _shifter)) == _bmWordSize,
+ "size inconsistency");
+ return _bmStartWord == (HeapWord*)(rs.base()) &&
+ _bmWordSize == rs.size()>>LogHeapWordSize;
+}
+#endif
+
+void CMBitMap::clearAll() {
+ _bm.clear();
+ return;
+}
+
+void CMBitMap::markRange(MemRegion mr) {
+ mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
+ assert(!mr.is_empty(), "unexpected empty region");
+ assert((offsetToHeapWord(heapWordToOffset(mr.end())) ==
+ ((HeapWord *) mr.end())),
+ "markRange memory region end is not card aligned");
+ // convert address range into offset range
+ _bm.at_put_range(heapWordToOffset(mr.start()),
+ heapWordToOffset(mr.end()), true);
+}
+
+void CMBitMap::clearRange(MemRegion mr) {
+ mr.intersection(MemRegion(_bmStartWord, _bmWordSize));
+ assert(!mr.is_empty(), "unexpected empty region");
+ // convert address range into offset range
+ _bm.at_put_range(heapWordToOffset(mr.start()),
+ heapWordToOffset(mr.end()), false);
+}
+
+MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr,
+ HeapWord* end_addr) {
+ HeapWord* start = getNextMarkedWordAddress(addr);
+ start = MIN2(start, end_addr);
+ HeapWord* end = getNextUnmarkedWordAddress(start);
+ end = MIN2(end, end_addr);
+ assert(start <= end, "Consistency check");
+ MemRegion mr(start, end);
+ if (!mr.is_empty()) {
+ clearRange(mr);
+ }
+ return mr;
+}
+
+CMMarkStack::CMMarkStack(ConcurrentMark* cm) :
+ _base(NULL), _cm(cm)
+#ifdef ASSERT
+ , _drain_in_progress(false)
+ , _drain_in_progress_yields(false)
+#endif
+{}
+
+void CMMarkStack::allocate(size_t size) {
+ _base = NEW_C_HEAP_ARRAY(oop, size);
+ if (_base == NULL)
+ vm_exit_during_initialization("Failed to allocate "
+ "CM region mark stack");
+ _index = 0;
+ // QQQQ cast ...
+ _capacity = (jint) size;
+ _oops_do_bound = -1;
+ NOT_PRODUCT(_max_depth = 0);
+}
+
+CMMarkStack::~CMMarkStack() {
+ if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
+}
+
+void CMMarkStack::par_push(oop ptr) {
+ while (true) {
+ if (isFull()) {
+ _overflow = true;
+ return;
+ }
+ // Otherwise...
+ jint index = _index;
+ jint next_index = index+1;
+ jint res = Atomic::cmpxchg(next_index, &_index, index);
+ if (res == index) {
+ _base[index] = ptr;
+ // Note that we don't maintain this atomically. We could, but it
+ // doesn't seem necessary.
+ NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
+ return;
+ }
+ // Otherwise, we need to try again.
+ }
+}
+
+void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) {
+ while (true) {
+ if (isFull()) {
+ _overflow = true;
+ return;
+ }
+ // Otherwise...
+ jint index = _index;
+ jint next_index = index + n;
+ if (next_index > _capacity) {
+ _overflow = true;
+ return;
+ }
+ jint res = Atomic::cmpxchg(next_index, &_index, index);
+ if (res == index) {
+ for (int i = 0; i < n; i++) {
+ int ind = index + i;
+ assert(ind < _capacity, "By overflow test above.");
+ _base[ind] = ptr_arr[i];
+ }
+ NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index));
+ return;
+ }
+ // Otherwise, we need to try again.
+ }
+}
+
+
+void CMMarkStack::par_push_arr(oop* ptr_arr, int n) {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ jint start = _index;
+ jint next_index = start + n;
+ if (next_index > _capacity) {
+ _overflow = true;
+ return;
+ }
+ // Otherwise.
+ _index = next_index;
+ for (int i = 0; i < n; i++) {
+ int ind = start + i;
+ guarantee(ind < _capacity, "By overflow test above.");
+ _base[ind] = ptr_arr[i];
+ }
+}
+
+
+bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ jint index = _index;
+ if (index == 0) {
+ *n = 0;
+ return false;
+ } else {
+ int k = MIN2(max, index);
+ jint new_ind = index - k;
+ for (int j = 0; j < k; j++) {
+ ptr_arr[j] = _base[new_ind + j];
+ }
+ _index = new_ind;
+ *n = k;
+ return true;
+ }
+}
+
+
+CMRegionStack::CMRegionStack() : _base(NULL) {}
+
+void CMRegionStack::allocate(size_t size) {
+ _base = NEW_C_HEAP_ARRAY(MemRegion, size);
+ if (_base == NULL)
+ vm_exit_during_initialization("Failed to allocate "
+ "CM region mark stack");
+ _index = 0;
+ // QQQQ cast ...
+ _capacity = (jint) size;
+}
+
+CMRegionStack::~CMRegionStack() {
+ if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base);
+}
+
+void CMRegionStack::push(MemRegion mr) {
+ assert(mr.word_size() > 0, "Precondition");
+ while (true) {
+ if (isFull()) {
+ _overflow = true;
+ return;
+ }
+ // Otherwise...
+ jint index = _index;
+ jint next_index = index+1;
+ jint res = Atomic::cmpxchg(next_index, &_index, index);
+ if (res == index) {
+ _base[index] = mr;
+ return;
+ }
+ // Otherwise, we need to try again.
+ }
+}
+
+MemRegion CMRegionStack::pop() {
+ while (true) {
+ // Otherwise...
+ jint index = _index;
+
+ if (index == 0) {
+ return MemRegion();
+ }
+ jint next_index = index-1;
+ jint res = Atomic::cmpxchg(next_index, &_index, index);
+ if (res == index) {
+ MemRegion mr = _base[next_index];
+ if (mr.start() != NULL) {
+ tmp_guarantee_CM( mr.end() != NULL, "invariant" );
+ tmp_guarantee_CM( mr.word_size() > 0, "invariant" );
+ return mr;
+ } else {
+ // that entry was invalidated... let's skip it
+ tmp_guarantee_CM( mr.end() == NULL, "invariant" );
+ }
+ }
+ // Otherwise, we need to try again.
+ }
+}
+
+bool CMRegionStack::invalidate_entries_into_cset() {
+ bool result = false;
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ for (int i = 0; i < _oops_do_bound; ++i) {
+ MemRegion mr = _base[i];
+ if (mr.start() != NULL) {
+ tmp_guarantee_CM( mr.end() != NULL, "invariant");
+ tmp_guarantee_CM( mr.word_size() > 0, "invariant" );
+ HeapRegion* hr = g1h->heap_region_containing(mr.start());
+ tmp_guarantee_CM( hr != NULL, "invariant" );
+ if (hr->in_collection_set()) {
+ // The region points into the collection set
+ _base[i] = MemRegion();
+ result = true;
+ }
+ } else {
+ // that entry was invalidated... let's skip it
+ tmp_guarantee_CM( mr.end() == NULL, "invariant" );
+ }
+ }
+ return result;
+}
+
+template
+bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) {
+ assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after
+ || SafepointSynchronize::is_at_safepoint(),
+ "Drain recursion must be yield-safe.");
+ bool res = true;
+ debug_only(_drain_in_progress = true);
+ debug_only(_drain_in_progress_yields = yield_after);
+ while (!isEmpty()) {
+ oop newOop = pop();
+ assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop");
+ assert(newOop->is_oop(), "Expected an oop");
+ assert(bm == NULL || bm->isMarked((HeapWord*)newOop),
+ "only grey objects on this stack");
+ // iterate over the oops in this oop, marking and pushing
+ // the ones in CMS generation.
+ newOop->oop_iterate(cl);
+ if (yield_after && _cm->do_yield_check()) {
+ res = false; break;
+ }
+ }
+ debug_only(_drain_in_progress = false);
+ return res;
+}
+
+void CMMarkStack::oops_do(OopClosure* f) {
+ if (_index == 0) return;
+ assert(_oops_do_bound != -1 && _oops_do_bound <= _index,
+ "Bound must be set.");
+ for (int i = 0; i < _oops_do_bound; i++) {
+ f->do_oop(&_base[i]);
+ }
+ _oops_do_bound = -1;
+}
+
+bool ConcurrentMark::not_yet_marked(oop obj) const {
+ return (_g1h->is_obj_ill(obj)
+ || (_g1h->is_in_permanent(obj)
+ && !nextMarkBitMap()->isMarked((HeapWord*)obj)));
+}
+
+#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
+#pragma warning( disable:4355 ) // 'this' : used in base member initializer list
+#endif // _MSC_VER
+
+ConcurrentMark::ConcurrentMark(ReservedSpace rs,
+ int max_regions) :
+ _markBitMap1(rs, MinObjAlignment - 1),
+ _markBitMap2(rs, MinObjAlignment - 1),
+
+ _parallel_marking_threads(0),
+ _sleep_factor(0.0),
+ _marking_task_overhead(1.0),
+ _cleanup_sleep_factor(0.0),
+ _cleanup_task_overhead(1.0),
+ _region_bm(max_regions, false /* in_resource_area*/),
+ _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
+ CardTableModRefBS::card_shift,
+ false /* in_resource_area*/),
+ _prevMarkBitMap(&_markBitMap1),
+ _nextMarkBitMap(&_markBitMap2),
+ _at_least_one_mark_complete(false),
+
+ _markStack(this),
+ _regionStack(),
+ // _finger set in set_non_marking_state
+
+ _max_task_num(MAX2(ParallelGCThreads, (size_t)1)),
+ // _active_tasks set in set_non_marking_state
+ // _tasks set inside the constructor
+ _task_queues(new CMTaskQueueSet((int) _max_task_num)),
+ _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)),
+
+ _has_overflown(false),
+ _concurrent(false),
+
+ // _verbose_level set below
+
+ _init_times(),
+ _remark_times(), _remark_mark_times(), _remark_weak_ref_times(),
+ _cleanup_times(),
+ _total_counting_time(0.0),
+ _total_rs_scrub_time(0.0),
+
+ _parallel_workers(NULL),
+ _cleanup_co_tracker(G1CLGroup)
+{
+ CMVerboseLevel verbose_level =
+ (CMVerboseLevel) G1MarkingVerboseLevel;
+ if (verbose_level < no_verbose)
+ verbose_level = no_verbose;
+ if (verbose_level > high_verbose)
+ verbose_level = high_verbose;
+ _verbose_level = verbose_level;
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", "
+ "heap end = "PTR_FORMAT, _heap_start, _heap_end);
+
+ _markStack.allocate(G1CMStackSize);
+ _regionStack.allocate(G1CMRegionStackSize);
+
+ // Create & start a ConcurrentMark thread.
+ if (G1ConcMark) {
+ _cmThread = new ConcurrentMarkThread(this);
+ assert(cmThread() != NULL, "CM Thread should have been created");
+ assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm");
+ } else {
+ _cmThread = NULL;
+ }
+ _g1h = G1CollectedHeap::heap();
+ assert(CGC_lock != NULL, "Where's the CGC_lock?");
+ assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
+ assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
+
+ SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
+ satb_qs.set_buffer_size(G1SATBLogBufferSize);
+
+ int size = (int) MAX2(ParallelGCThreads, (size_t)1);
+ _par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size);
+ for (int i = 0 ; i < size; i++) {
+ _par_cleanup_thread_state[i] = new ParCleanupThreadState;
+ }
+
+ _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
+ _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
+
+ // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
+ _active_tasks = _max_task_num;
+ for (int i = 0; i < (int) _max_task_num; ++i) {
+ CMTaskQueue* task_queue = new CMTaskQueue();
+ task_queue->initialize();
+ _task_queues->register_queue(i, task_queue);
+
+ _tasks[i] = new CMTask(i, this, task_queue, _task_queues);
+ _accum_task_vtime[i] = 0.0;
+ }
+
+ if (ParallelMarkingThreads > ParallelGCThreads) {
+ vm_exit_during_initialization("Can't have more ParallelMarkingThreads "
+ "than ParallelGCThreads.");
+ }
+ if (ParallelGCThreads == 0) {
+ // if we are not running with any parallel GC threads we will not
+ // spawn any marking threads either
+ _parallel_marking_threads = 0;
+ _sleep_factor = 0.0;
+ _marking_task_overhead = 1.0;
+ } else {
+ if (ParallelMarkingThreads > 0) {
+ // notice that ParallelMarkingThreads overwrites G1MarkingOverheadPerc
+ // if both are set
+
+ _parallel_marking_threads = ParallelMarkingThreads;
+ _sleep_factor = 0.0;
+ _marking_task_overhead = 1.0;
+ } else if (G1MarkingOverheadPerc > 0) {
+ // we will calculate the number of parallel marking threads
+ // based on a target overhead with respect to the soft real-time
+ // goal
+
+ double marking_overhead = (double) G1MarkingOverheadPerc / 100.0;
+ double overall_cm_overhead =
+ (double) G1MaxPauseTimeMS * marking_overhead / (double) G1TimeSliceMS;
+ double cpu_ratio = 1.0 / (double) os::processor_count();
+ double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio);
+ double marking_task_overhead =
+ overall_cm_overhead / marking_thread_num *
+ (double) os::processor_count();
+ double sleep_factor =
+ (1.0 - marking_task_overhead) / marking_task_overhead;
+
+ _parallel_marking_threads = (size_t) marking_thread_num;
+ _sleep_factor = sleep_factor;
+ _marking_task_overhead = marking_task_overhead;
+ } else {
+ _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1);
+ _sleep_factor = 0.0;
+ _marking_task_overhead = 1.0;
+ }
+
+ if (parallel_marking_threads() > 1)
+ _cleanup_task_overhead = 1.0;
+ else
+ _cleanup_task_overhead = marking_task_overhead();
+ _cleanup_sleep_factor =
+ (1.0 - cleanup_task_overhead()) / cleanup_task_overhead();
+
+#if 0
+ gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads());
+ gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead());
+ gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor());
+ gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead());
+ gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor());
+#endif
+
+ guarantee( parallel_marking_threads() > 0, "peace of mind" );
+ _parallel_workers = new WorkGang("Parallel Marking Threads",
+ (int) parallel_marking_threads(), false, true);
+ if (_parallel_workers == NULL)
+ vm_exit_during_initialization("Failed necessary allocation.");
+ }
+
+ // so that the call below can read a sensible value
+ _heap_start = (HeapWord*) rs.base();
+ set_non_marking_state();
+}
+
+void ConcurrentMark::update_g1_committed(bool force) {
+ // If concurrent marking is not in progress, then we do not need to
+ // update _heap_end. This has a subtle and important
+ // side-effect. Imagine that two evacuation pauses happen between
+ // marking completion and remark. The first one can grow the
+ // heap (hence now the finger is below the heap end). Then, the
+ // second one could unnecessarily push regions on the region
+ // stack. This causes the invariant that the region stack is empty
+ // at the beginning of remark to be false. By ensuring that we do
+ // not observe heap expansions after marking is complete, then we do
+ // not have this problem.
+ if (!concurrent_marking_in_progress() && !force)
+ return;
+
+ MemRegion committed = _g1h->g1_committed();
+ tmp_guarantee_CM( committed.start() == _heap_start,
+ "start shouldn't change" );
+ HeapWord* new_end = committed.end();
+ if (new_end > _heap_end) {
+ // The heap has been expanded.
+
+ _heap_end = new_end;
+ }
+ // Notice that the heap can also shrink. However, this only happens
+ // during a Full GC (at least currently) and the entire marking
+ // phase will bail out and the task will not be restarted. So, let's
+ // do nothing.
+}
+
+void ConcurrentMark::reset() {
+ // Starting values for these two. This should be called in a STW
+ // phase. CM will be notified of any future g1_committed expansions
+ // will be at the end of evacuation pauses, when tasks are
+ // inactive.
+ MemRegion committed = _g1h->g1_committed();
+ _heap_start = committed.start();
+ _heap_end = committed.end();
+
+ guarantee( _heap_start != NULL &&
+ _heap_end != NULL &&
+ _heap_start < _heap_end, "heap bounds should look ok" );
+
+ // reset all the marking data structures and any necessary flags
+ clear_marking_state();
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] resetting");
+
+ // We do reset all of them, since different phases will use
+ // different number of active threads. So, it's easiest to have all
+ // of them ready.
+ for (int i = 0; i < (int) _max_task_num; ++i)
+ _tasks[i]->reset(_nextMarkBitMap);
+
+ // we need this to make sure that the flag is on during the evac
+ // pause with initial mark piggy-backed
+ set_concurrent_marking_in_progress();
+}
+
+void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) {
+ guarantee( active_tasks <= _max_task_num, "we should not have more" );
+
+ _active_tasks = active_tasks;
+ // Need to update the three data structures below according to the
+ // number of active threads for this phase.
+ _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues);
+ _first_overflow_barrier_sync.set_n_workers((int) active_tasks);
+ _second_overflow_barrier_sync.set_n_workers((int) active_tasks);
+
+ _concurrent = concurrent;
+ // We propagate this to all tasks, not just the active ones.
+ for (int i = 0; i < (int) _max_task_num; ++i)
+ _tasks[i]->set_concurrent(concurrent);
+
+ if (concurrent) {
+ set_concurrent_marking_in_progress();
+ } else {
+ // We currently assume that the concurrent flag has been set to
+ // false before we start remark. At this point we should also be
+ // in a STW phase.
+ guarantee( !concurrent_marking_in_progress(), "invariant" );
+ guarantee( _finger == _heap_end, "only way to get here" );
+ update_g1_committed(true);
+ }
+}
+
+void ConcurrentMark::set_non_marking_state() {
+ // We set the global marking state to some default values when we're
+ // not doing marking.
+ clear_marking_state();
+ _active_tasks = 0;
+ clear_concurrent_marking_in_progress();
+}
+
+ConcurrentMark::~ConcurrentMark() {
+ int size = (int) MAX2(ParallelGCThreads, (size_t)1);
+ for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i];
+ FREE_C_HEAP_ARRAY(ParCleanupThreadState*,
+ _par_cleanup_thread_state);
+
+ for (int i = 0; i < (int) _max_task_num; ++i) {
+ delete _task_queues->queue(i);
+ delete _tasks[i];
+ }
+ delete _task_queues;
+ FREE_C_HEAP_ARRAY(CMTask*, _max_task_num);
+}
+
+// This closure is used to mark refs into the g1 generation
+// from external roots in the CMS bit map.
+// Called at the first checkpoint.
+//
+
+#define PRINT_REACHABLE_AT_INITIAL_MARK 0
+#if PRINT_REACHABLE_AT_INITIAL_MARK
+static FILE* reachable_file = NULL;
+
+class PrintReachableClosure: public OopsInGenClosure {
+ CMBitMap* _bm;
+ int _level;
+public:
+ PrintReachableClosure(CMBitMap* bm) :
+ _bm(bm), _level(0) {
+ guarantee(reachable_file != NULL, "pre-condition");
+ }
+ void do_oop(oop* p) {
+ oop obj = *p;
+ HeapWord* obj_addr = (HeapWord*)obj;
+ if (obj == NULL) return;
+ fprintf(reachable_file, "%d: "PTR_FORMAT" -> "PTR_FORMAT" (%d)\n",
+ _level, p, (void*) obj, _bm->isMarked(obj_addr));
+ if (!_bm->isMarked(obj_addr)) {
+ _bm->mark(obj_addr);
+ _level++;
+ obj->oop_iterate(this);
+ _level--;
+ }
+ }
+};
+#endif // PRINT_REACHABLE_AT_INITIAL_MARK
+
+#define SEND_HEAP_DUMP_TO_FILE 0
+#if SEND_HEAP_DUMP_TO_FILE
+static FILE* heap_dump_file = NULL;
+#endif // SEND_HEAP_DUMP_TO_FILE
+
+void ConcurrentMark::clearNextBitmap() {
+ guarantee(!G1CollectedHeap::heap()->mark_in_progress(), "Precondition.");
+
+ // clear the mark bitmap (no grey objects to start with).
+ // We need to do this in chunks and offer to yield in between
+ // each chunk.
+ HeapWord* start = _nextMarkBitMap->startWord();
+ HeapWord* end = _nextMarkBitMap->endWord();
+ HeapWord* cur = start;
+ size_t chunkSize = M;
+ while (cur < end) {
+ HeapWord* next = cur + chunkSize;
+ if (next > end)
+ next = end;
+ MemRegion mr(cur,next);
+ _nextMarkBitMap->clearRange(mr);
+ cur = next;
+ do_yield_check();
+ }
+}
+
+class NoteStartOfMarkHRClosure: public HeapRegionClosure {
+public:
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->continuesHumongous()) {
+ r->note_start_of_marking(true);
+ }
+ return false;
+ }
+};
+
+void ConcurrentMark::checkpointRootsInitialPre() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ G1CollectorPolicy* g1p = g1h->g1_policy();
+
+ _has_aborted = false;
+
+ // Find all the reachable objects...
+#if PRINT_REACHABLE_AT_INITIAL_MARK
+ guarantee(reachable_file == NULL, "Protocol");
+ char fn_buf[100];
+ sprintf(fn_buf, "/tmp/reachable.txt.%d", os::current_process_id());
+ reachable_file = fopen(fn_buf, "w");
+ // clear the mark bitmap (no grey objects to start with)
+ _nextMarkBitMap->clearAll();
+ PrintReachableClosure prcl(_nextMarkBitMap);
+ g1h->process_strong_roots(
+ false, // fake perm gen collection
+ SharedHeap::SO_AllClasses,
+ &prcl, // Regular roots
+ &prcl // Perm Gen Roots
+ );
+ // The root iteration above "consumed" dirty cards in the perm gen.
+ // Therefore, as a shortcut, we dirty all such cards.
+ g1h->rem_set()->invalidate(g1h->perm_gen()->used_region(), false);
+ fclose(reachable_file);
+ reachable_file = NULL;
+ // clear the mark bitmap again.
+ _nextMarkBitMap->clearAll();
+ COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+ COMPILER2_PRESENT(DerivedPointerTable::clear());
+#endif // PRINT_REACHABLE_AT_INITIAL_MARK
+
+ // Initialise marking structures. This has to be done in a STW phase.
+ reset();
+}
+
+class CMMarkRootsClosure: public OopsInGenClosure {
+private:
+ ConcurrentMark* _cm;
+ G1CollectedHeap* _g1h;
+ bool _do_barrier;
+
+public:
+ CMMarkRootsClosure(ConcurrentMark* cm,
+ G1CollectedHeap* g1h,
+ bool do_barrier) : _cm(cm), _g1h(g1h),
+ _do_barrier(do_barrier) { }
+
+ virtual void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ virtual void do_oop(oop* p) {
+ oop thisOop = *p;
+ if (thisOop != NULL) {
+ assert(thisOop->is_oop() || thisOop->mark() == NULL,
+ "expected an oop, possibly with mark word displaced");
+ HeapWord* addr = (HeapWord*)thisOop;
+ if (_g1h->is_in_g1_reserved(addr)) {
+ _cm->grayRoot(thisOop);
+ }
+ }
+ if (_do_barrier) {
+ assert(!_g1h->is_in_g1_reserved(p),
+ "Should be called on external roots");
+ do_barrier(p);
+ }
+ }
+};
+
+void ConcurrentMark::checkpointRootsInitialPost() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ // For each region note start of marking.
+ NoteStartOfMarkHRClosure startcl;
+ g1h->heap_region_iterate(&startcl);
+
+ // Start weak-reference discovery.
+ ReferenceProcessor* rp = g1h->ref_processor();
+ rp->verify_no_references_recorded();
+ rp->enable_discovery(); // enable ("weak") refs discovery
+ rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle
+
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ satb_mq_set.set_process_completed_threshold(G1SATBProcessCompletedThreshold);
+ satb_mq_set.set_active_all_threads(true);
+
+ // update_g1_committed() will be called at the end of an evac pause
+ // when marking is on. So, it's also called at the end of the
+ // initial-mark pause to update the heap end, if the heap expands
+ // during it. No need to call it here.
+
+ guarantee( !_cleanup_co_tracker.enabled(), "invariant" );
+
+ size_t max_marking_threads =
+ MAX2((size_t) 1, parallel_marking_threads());
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ _tasks[i]->enable_co_tracker();
+ if (i < (int) max_marking_threads)
+ _tasks[i]->reset_co_tracker(marking_task_overhead());
+ else
+ _tasks[i]->reset_co_tracker(0.0);
+ }
+}
+
+// Checkpoint the roots into this generation from outside
+// this generation. [Note this initial checkpoint need only
+// be approximate -- we'll do a catch up phase subsequently.]
+void ConcurrentMark::checkpointRootsInitial() {
+ assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped");
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ double start = os::elapsedTime();
+ GCOverheadReporter::recordSTWStart(start);
+
+ // If there has not been a GC[n-1] since last GC[n] cycle completed,
+ // precede our marking with a collection of all
+ // younger generations to keep floating garbage to a minimum.
+ // YSR: we won't do this for now -- it's an optimization to be
+ // done post-beta.
+
+ // YSR: ignoring weak refs for now; will do at bug fixing stage
+ // EVM: assert(discoveredRefsAreClear());
+
+
+ G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
+ g1p->record_concurrent_mark_init_start();
+ checkpointRootsInitialPre();
+
+ // YSR: when concurrent precleaning is in place, we'll
+ // need to clear the cached card table here
+
+ ResourceMark rm;
+ HandleMark hm;
+
+ g1h->ensure_parsability(false);
+ g1h->perm_gen()->save_marks();
+
+ CMMarkRootsClosure notOlder(this, g1h, false);
+ CMMarkRootsClosure older(this, g1h, true);
+
+ g1h->set_marking_started();
+ g1h->rem_set()->prepare_for_younger_refs_iterate(false);
+
+ g1h->process_strong_roots(false, // fake perm gen collection
+ SharedHeap::SO_AllClasses,
+ ¬Older, // Regular roots
+ &older // Perm Gen Roots
+ );
+ checkpointRootsInitialPost();
+
+ // Statistics.
+ double end = os::elapsedTime();
+ _init_times.add((end - start) * 1000.0);
+ GCOverheadReporter::recordSTWEnd(end);
+
+ g1p->record_concurrent_mark_init_end();
+}
+
+/*
+ Notice that in the next two methods, we actually leave the STS
+ during the barrier sync and join it immediately afterwards. If we
+ do not do this, this then the following deadlock can occur: one
+ thread could be in the barrier sync code, waiting for the other
+ thread to also sync up, whereas another one could be trying to
+ yield, while also waiting for the other threads to sync up too.
+
+ Because the thread that does the sync barrier has left the STS, it
+ is possible to be suspended for a Full GC or an evacuation pause
+ could occur. This is actually safe, since the entering the sync
+ barrier is one of the last things do_marking_step() does, and it
+ doesn't manipulate any data structures afterwards.
+*/
+
+void ConcurrentMark::enter_first_sync_barrier(int task_num) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] entering first barrier", task_num);
+
+ ConcurrentGCThread::stsLeave();
+ _first_overflow_barrier_sync.enter();
+ ConcurrentGCThread::stsJoin();
+ // at this point everyone should have synced up and not be doing any
+ // more work
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] leaving first barrier", task_num);
+
+ // let task 0 do this
+ if (task_num == 0) {
+ // task 0 is responsible for clearing the global data structures
+ clear_marking_state();
+
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]");
+ }
+ }
+
+ // after this, each task should reset its own data structures then
+ // then go into the second barrier
+}
+
+void ConcurrentMark::enter_second_sync_barrier(int task_num) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] entering second barrier", task_num);
+
+ ConcurrentGCThread::stsLeave();
+ _second_overflow_barrier_sync.enter();
+ ConcurrentGCThread::stsJoin();
+ // at this point everything should be re-initialised and ready to go
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] leaving second barrier", task_num);
+}
+
+void ConcurrentMark::grayRoot(oop p) {
+ HeapWord* addr = (HeapWord*) p;
+ // We can't really check against _heap_start and _heap_end, since it
+ // is possible during an evacuation pause with piggy-backed
+ // initial-mark that the committed space is expanded during the
+ // pause without CM observing this change. So the assertions below
+ // is a bit conservative; but better than nothing.
+ tmp_guarantee_CM( _g1h->g1_committed().contains(addr),
+ "address should be within the heap bounds" );
+
+ if (!_nextMarkBitMap->isMarked(addr))
+ _nextMarkBitMap->parMark(addr);
+}
+
+void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) {
+ // The objects on the region have already been marked "in bulk" by
+ // the caller. We only need to decide whether to push the region on
+ // the region stack or not.
+
+ if (!concurrent_marking_in_progress() || !_should_gray_objects)
+ // We're done with marking and waiting for remark. We do not need to
+ // push anything else on the region stack.
+ return;
+
+ HeapWord* finger = _finger;
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] attempting to push "
+ "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at "
+ PTR_FORMAT, mr.start(), mr.end(), finger);
+
+ if (mr.start() < finger) {
+ // The finger is always heap region aligned and it is not possible
+ // for mr to span heap regions.
+ tmp_guarantee_CM( mr.end() <= finger, "invariant" );
+
+ tmp_guarantee_CM( mr.start() <= mr.end() &&
+ _heap_start <= mr.start() &&
+ mr.end() <= _heap_end,
+ "region boundaries should fall within the committed space" );
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") "
+ "below the finger, pushing it",
+ mr.start(), mr.end());
+
+ if (!region_stack_push(mr)) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] region stack has overflown.");
+ }
+ }
+}
+
+void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) {
+ // The object is not marked by the caller. We need to at least mark
+ // it and maybe push in on the stack.
+
+ HeapWord* addr = (HeapWord*)p;
+ if (!_nextMarkBitMap->isMarked(addr)) {
+ // We definitely need to mark it, irrespective whether we bail out
+ // because we're done with marking.
+ if (_nextMarkBitMap->parMark(addr)) {
+ if (!concurrent_marking_in_progress() || !_should_gray_objects)
+ // If we're done with concurrent marking and we're waiting for
+ // remark, then we're not pushing anything on the stack.
+ return;
+
+ // No OrderAccess:store_load() is needed. It is implicit in the
+ // CAS done in parMark(addr) above
+ HeapWord* finger = _finger;
+
+ if (addr < finger) {
+ if (!mark_stack_push(oop(addr))) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] global stack overflow "
+ "during parMark");
+ }
+ }
+ }
+ }
+}
+
+class CMConcurrentMarkingTask: public AbstractGangTask {
+private:
+ ConcurrentMark* _cm;
+ ConcurrentMarkThread* _cmt;
+
+public:
+ void work(int worker_i) {
+ guarantee( Thread::current()->is_ConcurrentGC_thread(),
+ "this should only be done by a conc GC thread" );
+
+ double start_vtime = os::elapsedVTime();
+
+ ConcurrentGCThread::stsJoin();
+
+ guarantee( (size_t)worker_i < _cm->active_tasks(), "invariant" );
+ CMTask* the_task = _cm->task(worker_i);
+ the_task->start_co_tracker();
+ the_task->record_start_time();
+ if (!_cm->has_aborted()) {
+ do {
+ double start_vtime_sec = os::elapsedVTime();
+ double start_time_sec = os::elapsedTime();
+ the_task->do_marking_step(10.0);
+ double end_time_sec = os::elapsedTime();
+ double end_vtime_sec = os::elapsedVTime();
+ double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
+ double elapsed_time_sec = end_time_sec - start_time_sec;
+ _cm->clear_has_overflown();
+
+ bool ret = _cm->do_yield_check(worker_i);
+
+ jlong sleep_time_ms;
+ if (!_cm->has_aborted() && the_task->has_aborted()) {
+ sleep_time_ms =
+ (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0);
+ ConcurrentGCThread::stsLeave();
+ os::sleep(Thread::current(), sleep_time_ms, false);
+ ConcurrentGCThread::stsJoin();
+ }
+ double end_time2_sec = os::elapsedTime();
+ double elapsed_time2_sec = end_time2_sec - start_time_sec;
+
+ the_task->update_co_tracker();
+
+#if 0
+ gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, "
+ "overhead %1.4lf",
+ elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
+ the_task->conc_overhead(os::elapsedTime()) * 8.0);
+ gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms",
+ elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0);
+#endif
+ } while (!_cm->has_aborted() && the_task->has_aborted());
+ }
+ the_task->record_end_time();
+ guarantee( !the_task->has_aborted() || _cm->has_aborted(), "invariant" );
+
+ ConcurrentGCThread::stsLeave();
+
+ double end_vtime = os::elapsedVTime();
+ the_task->update_co_tracker(true);
+ _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime);
+ }
+
+ CMConcurrentMarkingTask(ConcurrentMark* cm,
+ ConcurrentMarkThread* cmt) :
+ AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { }
+
+ ~CMConcurrentMarkingTask() { }
+};
+
+void ConcurrentMark::markFromRoots() {
+ // we might be tempted to assert that:
+ // assert(asynch == !SafepointSynchronize::is_at_safepoint(),
+ // "inconsistent argument?");
+ // However that wouldn't be right, because it's possible that
+ // a safepoint is indeed in progress as a younger generation
+ // stop-the-world GC happens even as we mark in this generation.
+
+ _restart_for_overflow = false;
+
+ set_phase(MAX2((size_t) 1, parallel_marking_threads()), true);
+
+ CMConcurrentMarkingTask markingTask(this, cmThread());
+ if (parallel_marking_threads() > 0)
+ _parallel_workers->run_task(&markingTask);
+ else
+ markingTask.work(0);
+ print_stats();
+}
+
+void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) {
+ // world is stopped at this checkpoint
+ assert(SafepointSynchronize::is_at_safepoint(),
+ "world should be stopped");
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ // If a full collection has happened, we shouldn't do this.
+ if (has_aborted()) {
+ g1h->set_marking_complete(); // So bitmap clearing isn't confused
+ return;
+ }
+
+ G1CollectorPolicy* g1p = g1h->g1_policy();
+ g1p->record_concurrent_mark_remark_start();
+
+ double start = os::elapsedTime();
+ GCOverheadReporter::recordSTWStart(start);
+
+ checkpointRootsFinalWork();
+
+ double mark_work_end = os::elapsedTime();
+
+ weakRefsWork(clear_all_soft_refs);
+
+ if (has_overflown()) {
+ // Oops. We overflowed. Restart concurrent marking.
+ _restart_for_overflow = true;
+ // Clear the flag. We do not need it any more.
+ clear_has_overflown();
+ if (G1TraceMarkStackOverflow)
+ gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
+ } else {
+ // We're done with marking.
+ JavaThread::satb_mark_queue_set().set_active_all_threads(false);
+ }
+
+#if VERIFY_OBJS_PROCESSED
+ _scan_obj_cl.objs_processed = 0;
+ ThreadLocalObjQueue::objs_enqueued = 0;
+#endif
+
+ // Statistics
+ double now = os::elapsedTime();
+ _remark_mark_times.add((mark_work_end - start) * 1000.0);
+ _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
+ _remark_times.add((now - start) * 1000.0);
+
+ GCOverheadReporter::recordSTWEnd(now);
+ for (int i = 0; i < (int)_max_task_num; ++i)
+ _tasks[i]->disable_co_tracker();
+ _cleanup_co_tracker.enable();
+ _cleanup_co_tracker.reset(cleanup_task_overhead());
+ g1p->record_concurrent_mark_remark_end();
+}
+
+
+#define CARD_BM_TEST_MODE 0
+
+class CalcLiveObjectsClosure: public HeapRegionClosure {
+
+ CMBitMapRO* _bm;
+ ConcurrentMark* _cm;
+ COTracker* _co_tracker;
+ bool _changed;
+ bool _yield;
+ size_t _words_done;
+ size_t _tot_live;
+ size_t _tot_used;
+ size_t _regions_done;
+ double _start_vtime_sec;
+
+ BitMap* _region_bm;
+ BitMap* _card_bm;
+ intptr_t _bottom_card_num;
+ bool _final;
+
+ void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) {
+ for (intptr_t i = start_card_num; i <= last_card_num; i++) {
+#if CARD_BM_TEST_MODE
+ guarantee(_card_bm->at(i - _bottom_card_num),
+ "Should already be set.");
+#else
+ _card_bm->par_at_put(i - _bottom_card_num, 1);
+#endif
+ }
+ }
+
+public:
+ CalcLiveObjectsClosure(bool final,
+ CMBitMapRO *bm, ConcurrentMark *cm,
+ BitMap* region_bm, BitMap* card_bm,
+ COTracker* co_tracker) :
+ _bm(bm), _cm(cm), _changed(false), _yield(true),
+ _words_done(0), _tot_live(0), _tot_used(0),
+ _region_bm(region_bm), _card_bm(card_bm),
+ _final(final), _co_tracker(co_tracker),
+ _regions_done(0), _start_vtime_sec(0.0)
+ {
+ _bottom_card_num =
+ intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >>
+ CardTableModRefBS::card_shift);
+ }
+
+ bool doHeapRegion(HeapRegion* hr) {
+ if (_co_tracker != NULL)
+ _co_tracker->update();
+
+ if (!_final && _regions_done == 0)
+ _start_vtime_sec = os::elapsedVTime();
+
+ if (hr->continuesHumongous()) return false;
+
+ HeapWord* nextTop = hr->next_top_at_mark_start();
+ HeapWord* start = hr->top_at_conc_mark_count();
+ assert(hr->bottom() <= start && start <= hr->end() &&
+ hr->bottom() <= nextTop && nextTop <= hr->end() &&
+ start <= nextTop,
+ "Preconditions.");
+ // Otherwise, record the number of word's we'll examine.
+ size_t words_done = (nextTop - start);
+ // Find the first marked object at or after "start".
+ start = _bm->getNextMarkedWordAddress(start, nextTop);
+ size_t marked_bytes = 0;
+
+ // Below, the term "card num" means the result of shifting an address
+ // by the card shift -- address 0 corresponds to card number 0. One
+ // must subtract the card num of the bottom of the heap to obtain a
+ // card table index.
+ // The first card num of the sequence of live cards currently being
+ // constructed. -1 ==> no sequence.
+ intptr_t start_card_num = -1;
+ // The last card num of the sequence of live cards currently being
+ // constructed. -1 ==> no sequence.
+ intptr_t last_card_num = -1;
+
+ while (start < nextTop) {
+ if (_yield && _cm->do_yield_check()) {
+ // We yielded. It might be for a full collection, in which case
+ // all bets are off; terminate the traversal.
+ if (_cm->has_aborted()) {
+ _changed = false;
+ return true;
+ } else {
+ // Otherwise, it might be a collection pause, and the region
+ // we're looking at might be in the collection set. We'll
+ // abandon this region.
+ return false;
+ }
+ }
+ oop obj = oop(start);
+ int obj_sz = obj->size();
+ // The card num of the start of the current object.
+ intptr_t obj_card_num =
+ intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift);
+
+ HeapWord* obj_last = start + obj_sz - 1;
+ intptr_t obj_last_card_num =
+ intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift);
+
+ if (obj_card_num != last_card_num) {
+ if (start_card_num == -1) {
+ assert(last_card_num == -1, "Both or neither.");
+ start_card_num = obj_card_num;
+ } else {
+ assert(last_card_num != -1, "Both or neither.");
+ assert(obj_card_num >= last_card_num, "Inv");
+ if ((obj_card_num - last_card_num) > 1) {
+ // Mark the last run, and start a new one.
+ mark_card_num_range(start_card_num, last_card_num);
+ start_card_num = obj_card_num;
+ }
+ }
+#if CARD_BM_TEST_MODE
+ /*
+ gclog_or_tty->print_cr("Setting bits from %d/%d.",
+ obj_card_num - _bottom_card_num,
+ obj_last_card_num - _bottom_card_num);
+ */
+ for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) {
+ _card_bm->par_at_put(j - _bottom_card_num, 1);
+ }
+#endif
+ }
+ // In any case, we set the last card num.
+ last_card_num = obj_last_card_num;
+
+ marked_bytes += obj_sz * HeapWordSize;
+ // Find the next marked object after this one.
+ start = _bm->getNextMarkedWordAddress(start + 1, nextTop);
+ _changed = true;
+ }
+ // Handle the last range, if any.
+ if (start_card_num != -1)
+ mark_card_num_range(start_card_num, last_card_num);
+ if (_final) {
+ // Mark the allocated-since-marking portion...
+ HeapWord* tp = hr->top();
+ if (nextTop < tp) {
+ start_card_num =
+ intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift);
+ last_card_num =
+ intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift);
+ mark_card_num_range(start_card_num, last_card_num);
+ // This definitely means the region has live objects.
+ _region_bm->par_at_put(hr->hrs_index(), 1);
+ }
+ }
+
+ hr->add_to_marked_bytes(marked_bytes);
+ // Update the live region bitmap.
+ if (marked_bytes > 0) {
+ _region_bm->par_at_put(hr->hrs_index(), 1);
+ }
+ hr->set_top_at_conc_mark_count(nextTop);
+ _tot_live += hr->next_live_bytes();
+ _tot_used += hr->used();
+ _words_done = words_done;
+
+ if (!_final) {
+ ++_regions_done;
+ if (_regions_done % 10 == 0) {
+ double end_vtime_sec = os::elapsedVTime();
+ double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec;
+ if (elapsed_vtime_sec > (10.0 / 1000.0)) {
+ jlong sleep_time_ms =
+ (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0);
+#if 0
+ gclog_or_tty->print_cr("CL: elapsed %1.4lf ms, sleep %1.4lf ms, "
+ "overhead %1.4lf",
+ elapsed_vtime_sec * 1000.0, (double) sleep_time_ms,
+ _co_tracker->concOverhead(os::elapsedTime()));
+#endif
+ os::sleep(Thread::current(), sleep_time_ms, false);
+ _start_vtime_sec = end_vtime_sec;
+ }
+ }
+ }
+
+ return false;
+ }
+
+ bool changed() { return _changed; }
+ void reset() { _changed = false; _words_done = 0; }
+ void no_yield() { _yield = false; }
+ size_t words_done() { return _words_done; }
+ size_t tot_live() { return _tot_live; }
+ size_t tot_used() { return _tot_used; }
+};
+
+
+void ConcurrentMark::calcDesiredRegions() {
+ guarantee( _cleanup_co_tracker.enabled(), "invariant" );
+ _cleanup_co_tracker.start();
+
+ _region_bm.clear();
+ _card_bm.clear();
+ CalcLiveObjectsClosure calccl(false /*final*/,
+ nextMarkBitMap(), this,
+ &_region_bm, &_card_bm,
+ &_cleanup_co_tracker);
+ G1CollectedHeap *g1h = G1CollectedHeap::heap();
+ g1h->heap_region_iterate(&calccl);
+
+ do {
+ calccl.reset();
+ g1h->heap_region_iterate(&calccl);
+ } while (calccl.changed());
+
+ _cleanup_co_tracker.update(true);
+}
+
+class G1ParFinalCountTask: public AbstractGangTask {
+protected:
+ G1CollectedHeap* _g1h;
+ CMBitMap* _bm;
+ size_t _n_workers;
+ size_t *_live_bytes;
+ size_t *_used_bytes;
+ BitMap* _region_bm;
+ BitMap* _card_bm;
+public:
+ G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm,
+ BitMap* region_bm, BitMap* card_bm) :
+ AbstractGangTask("G1 final counting"), _g1h(g1h),
+ _bm(bm), _region_bm(region_bm), _card_bm(card_bm)
+ {
+ if (ParallelGCThreads > 0)
+ _n_workers = _g1h->workers()->total_workers();
+ else
+ _n_workers = 1;
+ _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
+ _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers);
+ }
+
+ ~G1ParFinalCountTask() {
+ FREE_C_HEAP_ARRAY(size_t, _live_bytes);
+ FREE_C_HEAP_ARRAY(size_t, _used_bytes);
+ }
+
+ void work(int i) {
+ CalcLiveObjectsClosure calccl(true /*final*/,
+ _bm, _g1h->concurrent_mark(),
+ _region_bm, _card_bm,
+ NULL /* CO tracker */);
+ calccl.no_yield();
+ if (ParallelGCThreads > 0) {
+ _g1h->heap_region_par_iterate_chunked(&calccl, i,
+ HeapRegion::FinalCountClaimValue);
+ } else {
+ _g1h->heap_region_iterate(&calccl);
+ }
+ assert(calccl.complete(), "Shouldn't have yielded!");
+
+ guarantee( (size_t)i < _n_workers, "invariant" );
+ _live_bytes[i] = calccl.tot_live();
+ _used_bytes[i] = calccl.tot_used();
+ }
+ size_t live_bytes() {
+ size_t live_bytes = 0;
+ for (size_t i = 0; i < _n_workers; ++i)
+ live_bytes += _live_bytes[i];
+ return live_bytes;
+ }
+ size_t used_bytes() {
+ size_t used_bytes = 0;
+ for (size_t i = 0; i < _n_workers; ++i)
+ used_bytes += _used_bytes[i];
+ return used_bytes;
+ }
+};
+
+class G1ParNoteEndTask;
+
+class G1NoteEndOfConcMarkClosure : public HeapRegionClosure {
+ G1CollectedHeap* _g1;
+ int _worker_num;
+ size_t _max_live_bytes;
+ size_t _regions_claimed;
+ size_t _freed_bytes;
+ size_t _cleared_h_regions;
+ size_t _freed_regions;
+ UncleanRegionList* _unclean_region_list;
+ double _claimed_region_time;
+ double _max_region_time;
+
+public:
+ G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
+ UncleanRegionList* list,
+ int worker_num);
+ size_t freed_bytes() { return _freed_bytes; }
+ size_t cleared_h_regions() { return _cleared_h_regions; }
+ size_t freed_regions() { return _freed_regions; }
+ UncleanRegionList* unclean_region_list() {
+ return _unclean_region_list;
+ }
+
+ bool doHeapRegion(HeapRegion *r);
+
+ size_t max_live_bytes() { return _max_live_bytes; }
+ size_t regions_claimed() { return _regions_claimed; }
+ double claimed_region_time_sec() { return _claimed_region_time; }
+ double max_region_time_sec() { return _max_region_time; }
+};
+
+class G1ParNoteEndTask: public AbstractGangTask {
+ friend class G1NoteEndOfConcMarkClosure;
+protected:
+ G1CollectedHeap* _g1h;
+ size_t _max_live_bytes;
+ size_t _freed_bytes;
+ ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state;
+public:
+ G1ParNoteEndTask(G1CollectedHeap* g1h,
+ ConcurrentMark::ParCleanupThreadState**
+ par_cleanup_thread_state) :
+ AbstractGangTask("G1 note end"), _g1h(g1h),
+ _max_live_bytes(0), _freed_bytes(0),
+ _par_cleanup_thread_state(par_cleanup_thread_state)
+ {}
+
+ void work(int i) {
+ double start = os::elapsedTime();
+ G1NoteEndOfConcMarkClosure g1_note_end(_g1h,
+ &_par_cleanup_thread_state[i]->list,
+ i);
+ if (ParallelGCThreads > 0) {
+ _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
+ HeapRegion::NoteEndClaimValue);
+ } else {
+ _g1h->heap_region_iterate(&g1_note_end);
+ }
+ assert(g1_note_end.complete(), "Shouldn't have yielded!");
+
+ // Now finish up freeing the current thread's regions.
+ _g1h->finish_free_region_work(g1_note_end.freed_bytes(),
+ g1_note_end.cleared_h_regions(),
+ 0, NULL);
+ {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ _max_live_bytes += g1_note_end.max_live_bytes();
+ _freed_bytes += g1_note_end.freed_bytes();
+ }
+ double end = os::elapsedTime();
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
+ "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
+ i, start, end, (end-start)*1000.0,
+ g1_note_end.regions_claimed(),
+ g1_note_end.claimed_region_time_sec()*1000.0,
+ g1_note_end.max_region_time_sec()*1000.0);
+ }
+ }
+ size_t max_live_bytes() { return _max_live_bytes; }
+ size_t freed_bytes() { return _freed_bytes; }
+};
+
+class G1ParScrubRemSetTask: public AbstractGangTask {
+protected:
+ G1RemSet* _g1rs;
+ BitMap* _region_bm;
+ BitMap* _card_bm;
+public:
+ G1ParScrubRemSetTask(G1CollectedHeap* g1h,
+ BitMap* region_bm, BitMap* card_bm) :
+ AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()),
+ _region_bm(region_bm), _card_bm(card_bm)
+ {}
+
+ void work(int i) {
+ if (ParallelGCThreads > 0) {
+ _g1rs->scrub_par(_region_bm, _card_bm, i,
+ HeapRegion::ScrubRemSetClaimValue);
+ } else {
+ _g1rs->scrub(_region_bm, _card_bm);
+ }
+ }
+
+};
+
+G1NoteEndOfConcMarkClosure::
+G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
+ UncleanRegionList* list,
+ int worker_num)
+ : _g1(g1), _worker_num(worker_num),
+ _max_live_bytes(0), _regions_claimed(0),
+ _freed_bytes(0), _cleared_h_regions(0), _freed_regions(0),
+ _claimed_region_time(0.0), _max_region_time(0.0),
+ _unclean_region_list(list)
+{}
+
+bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) {
+ // We use a claim value of zero here because all regions
+ // were claimed with value 1 in the FinalCount task.
+ r->reset_gc_time_stamp();
+ if (!r->continuesHumongous()) {
+ double start = os::elapsedTime();
+ _regions_claimed++;
+ r->note_end_of_marking();
+ _max_live_bytes += r->max_live_bytes();
+ _g1->free_region_if_totally_empty_work(r,
+ _freed_bytes,
+ _cleared_h_regions,
+ _freed_regions,
+ _unclean_region_list,
+ true /*par*/);
+ double region_time = (os::elapsedTime() - start);
+ _claimed_region_time += region_time;
+ if (region_time > _max_region_time) _max_region_time = region_time;
+ }
+ return false;
+}
+
+void ConcurrentMark::cleanup() {
+ // world is stopped at this checkpoint
+ assert(SafepointSynchronize::is_at_safepoint(),
+ "world should be stopped");
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ // If a full collection has happened, we shouldn't do this.
+ if (has_aborted()) {
+ g1h->set_marking_complete(); // So bitmap clearing isn't confused
+ return;
+ }
+
+ _cleanup_co_tracker.disable();
+
+ G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
+ g1p->record_concurrent_mark_cleanup_start();
+
+ double start = os::elapsedTime();
+ GCOverheadReporter::recordSTWStart(start);
+
+ // Do counting once more with the world stopped for good measure.
+ G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
+ &_region_bm, &_card_bm);
+ if (ParallelGCThreads > 0) {
+ assert(g1h->check_heap_region_claim_values(
+ HeapRegion::InitialClaimValue),
+ "sanity check");
+
+ int n_workers = g1h->workers()->total_workers();
+ g1h->set_par_threads(n_workers);
+ g1h->workers()->run_task(&g1_par_count_task);
+ g1h->set_par_threads(0);
+
+ assert(g1h->check_heap_region_claim_values(
+ HeapRegion::FinalCountClaimValue),
+ "sanity check");
+ } else {
+ g1_par_count_task.work(0);
+ }
+
+ size_t known_garbage_bytes =
+ g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes();
+#if 0
+ gclog_or_tty->print_cr("used %1.2lf, live %1.2lf, garbage %1.2lf",
+ (double) g1_par_count_task.used_bytes() / (double) (1024 * 1024),
+ (double) g1_par_count_task.live_bytes() / (double) (1024 * 1024),
+ (double) known_garbage_bytes / (double) (1024 * 1024));
+#endif // 0
+ g1p->set_known_garbage_bytes(known_garbage_bytes);
+
+ size_t start_used_bytes = g1h->used();
+ _at_least_one_mark_complete = true;
+ g1h->set_marking_complete();
+
+ double count_end = os::elapsedTime();
+ double this_final_counting_time = (count_end - start);
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print_cr("Cleanup:");
+ gclog_or_tty->print_cr(" Finalize counting: %8.3f ms",
+ this_final_counting_time*1000.0);
+ }
+ _total_counting_time += this_final_counting_time;
+
+ // Install newly created mark bitMap as "prev".
+ swapMarkBitMaps();
+
+ g1h->reset_gc_time_stamp();
+
+ // Note end of marking in all heap regions.
+ double note_end_start = os::elapsedTime();
+ G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state);
+ if (ParallelGCThreads > 0) {
+ int n_workers = g1h->workers()->total_workers();
+ g1h->set_par_threads(n_workers);
+ g1h->workers()->run_task(&g1_par_note_end_task);
+ g1h->set_par_threads(0);
+
+ assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
+ "sanity check");
+ } else {
+ g1_par_note_end_task.work(0);
+ }
+ g1h->set_unclean_regions_coming(true);
+ double note_end_end = os::elapsedTime();
+ // Tell the mutators that there might be unclean regions coming...
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print_cr(" note end of marking: %8.3f ms.",
+ (note_end_end - note_end_start)*1000.0);
+ }
+
+
+ // call below, since it affects the metric by which we sort the heap
+ // regions.
+ if (G1ScrubRemSets) {
+ double rs_scrub_start = os::elapsedTime();
+ G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm);
+ if (ParallelGCThreads > 0) {
+ int n_workers = g1h->workers()->total_workers();
+ g1h->set_par_threads(n_workers);
+ g1h->workers()->run_task(&g1_par_scrub_rs_task);
+ g1h->set_par_threads(0);
+
+ assert(g1h->check_heap_region_claim_values(
+ HeapRegion::ScrubRemSetClaimValue),
+ "sanity check");
+ } else {
+ g1_par_scrub_rs_task.work(0);
+ }
+
+ double rs_scrub_end = os::elapsedTime();
+ double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start);
+ _total_rs_scrub_time += this_rs_scrub_time;
+ }
+
+ // this will also free any regions totally full of garbage objects,
+ // and sort the regions.
+ g1h->g1_policy()->record_concurrent_mark_cleanup_end(
+ g1_par_note_end_task.freed_bytes(),
+ g1_par_note_end_task.max_live_bytes());
+
+ // Statistics.
+ double end = os::elapsedTime();
+ _cleanup_times.add((end - start) * 1000.0);
+ GCOverheadReporter::recordSTWEnd(end);
+
+ // G1CollectedHeap::heap()->print();
+ // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d",
+ // G1CollectedHeap::heap()->get_gc_time_stamp());
+
+ if (PrintGC || PrintGCDetails) {
+ g1h->print_size_transition(gclog_or_tty,
+ start_used_bytes,
+ g1h->used(),
+ g1h->capacity());
+ }
+
+ size_t cleaned_up_bytes = start_used_bytes - g1h->used();
+ g1p->decrease_known_garbage_bytes(cleaned_up_bytes);
+
+ // We need to make this be a "collection" so any collection pause that
+ // races with it goes around and waits for completeCleanup to finish.
+ g1h->increment_total_collections();
+
+#ifndef PRODUCT
+ if (G1VerifyConcMark) {
+ G1CollectedHeap::heap()->prepare_for_verify();
+ G1CollectedHeap::heap()->verify(true,false);
+ }
+#endif
+}
+
+void ConcurrentMark::completeCleanup() {
+ // A full collection intervened.
+ if (has_aborted()) return;
+
+ int first = 0;
+ int last = (int)MAX2(ParallelGCThreads, (size_t)1);
+ for (int t = 0; t < last; t++) {
+ UncleanRegionList* list = &_par_cleanup_thread_state[t]->list;
+ assert(list->well_formed(), "Inv");
+ HeapRegion* hd = list->hd();
+ while (hd != NULL) {
+ // Now finish up the other stuff.
+ hd->rem_set()->clear();
+ HeapRegion* next_hd = hd->next_from_unclean_list();
+ (void)list->pop();
+ guarantee(list->hd() == next_hd, "how not?");
+ _g1h->put_region_on_unclean_list(hd);
+ if (!hd->isHumongous()) {
+ // Add this to the _free_regions count by 1.
+ _g1h->finish_free_region_work(0, 0, 1, NULL);
+ }
+ hd = list->hd();
+ guarantee(hd == next_hd, "how not?");
+ }
+ }
+}
+
+
+class G1CMIsAliveClosure: public BoolObjectClosure {
+ G1CollectedHeap* _g1;
+ public:
+ G1CMIsAliveClosure(G1CollectedHeap* g1) :
+ _g1(g1)
+ {}
+
+ void do_object(oop obj) {
+ assert(false, "not to be invoked");
+ }
+ bool do_object_b(oop obj) {
+ HeapWord* addr = (HeapWord*)obj;
+ return addr != NULL &&
+ (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
+ }
+};
+
+class G1CMKeepAliveClosure: public OopClosure {
+ G1CollectedHeap* _g1;
+ ConcurrentMark* _cm;
+ CMBitMap* _bitMap;
+ public:
+ G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
+ CMBitMap* bitMap) :
+ _g1(g1), _cm(cm),
+ _bitMap(bitMap) {}
+
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ void do_oop(oop* p) {
+ oop thisOop = *p;
+ HeapWord* addr = (HeapWord*)thisOop;
+ if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
+ _bitMap->mark(addr);
+ _cm->mark_stack_push(thisOop);
+ }
+ }
+};
+
+class G1CMDrainMarkingStackClosure: public VoidClosure {
+ CMMarkStack* _markStack;
+ CMBitMap* _bitMap;
+ G1CMKeepAliveClosure* _oopClosure;
+ public:
+ G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack,
+ G1CMKeepAliveClosure* oopClosure) :
+ _bitMap(bitMap),
+ _markStack(markStack),
+ _oopClosure(oopClosure)
+ {}
+
+ void do_void() {
+ _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
+ }
+};
+
+void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
+ ResourceMark rm;
+ HandleMark hm;
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ ReferenceProcessor* rp = g1h->ref_processor();
+
+ // Process weak references.
+ rp->setup_policy(clear_all_soft_refs);
+ assert(_markStack.isEmpty(), "mark stack should be empty");
+
+ G1CMIsAliveClosure g1IsAliveClosure (g1h);
+ G1CMKeepAliveClosure g1KeepAliveClosure(g1h, this, nextMarkBitMap());
+ G1CMDrainMarkingStackClosure
+ g1DrainMarkingStackClosure(nextMarkBitMap(), &_markStack,
+ &g1KeepAliveClosure);
+
+ // XXXYYY Also: copy the parallel ref processing code from CMS.
+ rp->process_discovered_references(&g1IsAliveClosure,
+ &g1KeepAliveClosure,
+ &g1DrainMarkingStackClosure,
+ NULL);
+ assert(_markStack.overflow() || _markStack.isEmpty(),
+ "mark stack should be empty (unless it overflowed)");
+ if (_markStack.overflow()) {
+ set_has_overflown();
+ }
+
+ rp->enqueue_discovered_references();
+ rp->verify_no_references_recorded();
+ assert(!rp->discovery_enabled(), "should have been disabled");
+
+ // Now clean up stale oops in SymbolTable and StringTable
+ SymbolTable::unlink(&g1IsAliveClosure);
+ StringTable::unlink(&g1IsAliveClosure);
+}
+
+void ConcurrentMark::swapMarkBitMaps() {
+ CMBitMapRO* temp = _prevMarkBitMap;
+ _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap;
+ _nextMarkBitMap = (CMBitMap*) temp;
+}
+
+class CMRemarkTask: public AbstractGangTask {
+private:
+ ConcurrentMark *_cm;
+
+public:
+ void work(int worker_i) {
+ // Since all available tasks are actually started, we should
+ // only proceed if we're supposed to be actived.
+ if ((size_t)worker_i < _cm->active_tasks()) {
+ CMTask* task = _cm->task(worker_i);
+ task->record_start_time();
+ do {
+ task->do_marking_step(1000000000.0 /* something very large */);
+ } while (task->has_aborted() && !_cm->has_overflown());
+ // If we overflow, then we do not want to restart. We instead
+ // want to abort remark and do concurrent marking again.
+ task->record_end_time();
+ }
+ }
+
+ CMRemarkTask(ConcurrentMark* cm) :
+ AbstractGangTask("Par Remark"), _cm(cm) { }
+};
+
+void ConcurrentMark::checkpointRootsFinalWork() {
+ ResourceMark rm;
+ HandleMark hm;
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ g1h->ensure_parsability(false);
+
+ if (ParallelGCThreads > 0) {
+ g1h->change_strong_roots_parity();
+ // this is remark, so we'll use up all available threads
+ int active_workers = ParallelGCThreads;
+ set_phase(active_workers, false);
+
+ CMRemarkTask remarkTask(this);
+ // We will start all available threads, even if we decide that the
+ // active_workers will be fewer. The extra ones will just bail out
+ // immediately.
+ int n_workers = g1h->workers()->total_workers();
+ g1h->set_par_threads(n_workers);
+ g1h->workers()->run_task(&remarkTask);
+ g1h->set_par_threads(0);
+
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
+ } else {
+ g1h->change_strong_roots_parity();
+ // this is remark, so we'll use up all available threads
+ int active_workers = 1;
+ set_phase(active_workers, false);
+
+ CMRemarkTask remarkTask(this);
+ // We will start all available threads, even if we decide that the
+ // active_workers will be fewer. The extra ones will just bail out
+ // immediately.
+ remarkTask.work(0);
+
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
+ }
+
+ print_stats();
+
+ if (!restart_for_overflow())
+ set_non_marking_state();
+
+#if VERIFY_OBJS_PROCESSED
+ if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
+ gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
+ _scan_obj_cl.objs_processed,
+ ThreadLocalObjQueue::objs_enqueued);
+ guarantee(_scan_obj_cl.objs_processed ==
+ ThreadLocalObjQueue::objs_enqueued,
+ "Different number of objs processed and enqueued.");
+ }
+#endif
+}
+
+class ReachablePrinterOopClosure: public OopClosure {
+private:
+ G1CollectedHeap* _g1h;
+ CMBitMapRO* _bitmap;
+ outputStream* _out;
+
+public:
+ ReachablePrinterOopClosure(CMBitMapRO* bitmap, outputStream* out) :
+ _bitmap(bitmap), _g1h(G1CollectedHeap::heap()), _out(out) { }
+
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ void do_oop(oop* p) {
+ oop obj = *p;
+ const char* str = NULL;
+ const char* str2 = "";
+
+ if (!_g1h->is_in_g1_reserved(obj))
+ str = "outside G1 reserved";
+ else {
+ HeapRegion* hr = _g1h->heap_region_containing(obj);
+ guarantee( hr != NULL, "invariant" );
+ if (hr->obj_allocated_since_prev_marking(obj)) {
+ str = "over TAMS";
+ if (_bitmap->isMarked((HeapWord*) obj))
+ str2 = " AND MARKED";
+ } else if (_bitmap->isMarked((HeapWord*) obj))
+ str = "marked";
+ else
+ str = "#### NOT MARKED ####";
+ }
+
+ _out->print_cr(" "PTR_FORMAT" contains "PTR_FORMAT" %s%s",
+ p, (void*) obj, str, str2);
+ }
+};
+
+class ReachablePrinterClosure: public BitMapClosure {
+private:
+ CMBitMapRO* _bitmap;
+ outputStream* _out;
+
+public:
+ ReachablePrinterClosure(CMBitMapRO* bitmap, outputStream* out) :
+ _bitmap(bitmap), _out(out) { }
+
+ bool do_bit(size_t offset) {
+ HeapWord* addr = _bitmap->offsetToHeapWord(offset);
+ ReachablePrinterOopClosure oopCl(_bitmap, _out);
+
+ _out->print_cr(" obj "PTR_FORMAT", offset %10d (marked)", addr, offset);
+ oop(addr)->oop_iterate(&oopCl);
+ _out->print_cr("");
+
+ return true;
+ }
+};
+
+class ObjInRegionReachablePrinterClosure : public ObjectClosure {
+private:
+ CMBitMapRO* _bitmap;
+ outputStream* _out;
+
+public:
+ void do_object(oop o) {
+ ReachablePrinterOopClosure oopCl(_bitmap, _out);
+
+ _out->print_cr(" obj "PTR_FORMAT" (over TAMS)", (void*) o);
+ o->oop_iterate(&oopCl);
+ _out->print_cr("");
+ }
+
+ ObjInRegionReachablePrinterClosure(CMBitMapRO* bitmap, outputStream* out) :
+ _bitmap(bitmap), _out(out) { }
+};
+
+class RegionReachablePrinterClosure : public HeapRegionClosure {
+private:
+ CMBitMapRO* _bitmap;
+ outputStream* _out;
+
+public:
+ bool doHeapRegion(HeapRegion* hr) {
+ HeapWord* b = hr->bottom();
+ HeapWord* e = hr->end();
+ HeapWord* t = hr->top();
+ HeapWord* p = hr->prev_top_at_mark_start();
+ _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
+ "PTAMS: "PTR_FORMAT, b, e, t, p);
+ _out->print_cr("");
+
+ ObjInRegionReachablePrinterClosure ocl(_bitmap, _out);
+ hr->object_iterate_mem_careful(MemRegion(p, t), &ocl);
+
+ return false;
+ }
+
+ RegionReachablePrinterClosure(CMBitMapRO* bitmap,
+ outputStream* out) :
+ _bitmap(bitmap), _out(out) { }
+};
+
+void ConcurrentMark::print_prev_bitmap_reachable() {
+ outputStream* out = gclog_or_tty;
+
+#if SEND_HEAP_DUMP_TO_FILE
+ guarantee(heap_dump_file == NULL, "Protocol");
+ char fn_buf[100];
+ sprintf(fn_buf, "/tmp/dump.txt.%d", os::current_process_id());
+ heap_dump_file = fopen(fn_buf, "w");
+ fileStream fstream(heap_dump_file);
+ out = &fstream;
+#endif // SEND_HEAP_DUMP_TO_FILE
+
+ RegionReachablePrinterClosure rcl(_prevMarkBitMap, out);
+ out->print_cr("--- ITERATING OVER REGIONS WITH PTAMS < TOP");
+ _g1h->heap_region_iterate(&rcl);
+ out->print_cr("");
+
+ ReachablePrinterClosure cl(_prevMarkBitMap, out);
+ out->print_cr("--- REACHABLE OBJECTS ON THE BITMAP");
+ _prevMarkBitMap->iterate(&cl);
+ out->print_cr("");
+
+#if SEND_HEAP_DUMP_TO_FILE
+ fclose(heap_dump_file);
+ heap_dump_file = NULL;
+#endif // SEND_HEAP_DUMP_TO_FILE
+}
+
+// This note is for drainAllSATBBuffers and the code in between.
+// In the future we could reuse a task to do this work during an
+// evacuation pause (since now tasks are not active and can be claimed
+// during an evacuation pause). This was a late change to the code and
+// is currently not being taken advantage of.
+
+class CMGlobalObjectClosure : public ObjectClosure {
+private:
+ ConcurrentMark* _cm;
+
+public:
+ void do_object(oop obj) {
+ _cm->deal_with_reference(obj);
+ }
+
+ CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { }
+};
+
+void ConcurrentMark::deal_with_reference(oop obj) {
+ if (verbose_high())
+ gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT,
+ (void*) obj);
+
+
+ HeapWord* objAddr = (HeapWord*) obj;
+ if (_g1h->is_in_g1_reserved(objAddr)) {
+ tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
+ HeapRegion* hr = _g1h->heap_region_containing(obj);
+ if (_g1h->is_obj_ill(obj, hr)) {
+ if (verbose_high())
+ gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered "
+ "marked", (void*) obj);
+
+ // we need to mark it first
+ if (_nextMarkBitMap->parMark(objAddr)) {
+ // No OrderAccess:store_load() is needed. It is implicit in the
+ // CAS done in parMark(objAddr) above
+ HeapWord* finger = _finger;
+ if (objAddr < finger) {
+ if (verbose_high())
+ gclog_or_tty->print_cr("[global] below the global finger "
+ "("PTR_FORMAT"), pushing it", finger);
+ if (!mark_stack_push(obj)) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[global] global stack overflow during "
+ "deal_with_reference");
+ }
+ }
+ }
+ }
+ }
+}
+
+void ConcurrentMark::drainAllSATBBuffers() {
+ CMGlobalObjectClosure oc(this);
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ satb_mq_set.set_closure(&oc);
+
+ while (satb_mq_set.apply_closure_to_completed_buffer()) {
+ if (verbose_medium())
+ gclog_or_tty->print_cr("[global] processed an SATB buffer");
+ }
+
+ // no need to check whether we should do this, as this is only
+ // called during an evacuation pause
+ satb_mq_set.iterate_closure_all_threads();
+
+ satb_mq_set.set_closure(NULL);
+ guarantee( satb_mq_set.completed_buffers_num() == 0, "invariant" );
+}
+
+void ConcurrentMark::markPrev(oop p) {
+ // Note we are overriding the read-only view of the prev map here, via
+ // the cast.
+ ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p);
+}
+
+void ConcurrentMark::clear(oop p) {
+ assert(p != NULL && p->is_oop(), "expected an oop");
+ HeapWord* addr = (HeapWord*)p;
+ assert(addr >= _nextMarkBitMap->startWord() ||
+ addr < _nextMarkBitMap->endWord(), "in a region");
+
+ _nextMarkBitMap->clear(addr);
+}
+
+void ConcurrentMark::clearRangeBothMaps(MemRegion mr) {
+ // Note we are overriding the read-only view of the prev map here, via
+ // the cast.
+ ((CMBitMap*)_prevMarkBitMap)->clearRange(mr);
+ _nextMarkBitMap->clearRange(mr);
+}
+
+HeapRegion*
+ConcurrentMark::claim_region(int task_num) {
+ // "checkpoint" the finger
+ HeapWord* finger = _finger;
+
+ // _heap_end will not change underneath our feet; it only changes at
+ // yield points.
+ while (finger < _heap_end) {
+ tmp_guarantee_CM( _g1h->is_in_g1_reserved(finger), "invariant" );
+
+ // is the gap between reading the finger and doing the CAS too long?
+
+ HeapRegion* curr_region = _g1h->heap_region_containing(finger);
+ HeapWord* bottom = curr_region->bottom();
+ HeapWord* end = curr_region->end();
+ HeapWord* limit = curr_region->next_top_at_mark_start();
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" "
+ "["PTR_FORMAT", "PTR_FORMAT"), "
+ "limit = "PTR_FORMAT,
+ task_num, curr_region, bottom, end, limit);
+
+ HeapWord* res =
+ (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger);
+ if (res == finger) {
+ // we succeeded
+
+ // notice that _finger == end cannot be guaranteed here since,
+ // someone else might have moved the finger even further
+ guarantee( _finger >= end, "the finger should have moved forward" );
+
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] we were successful with region = "
+ PTR_FORMAT, task_num, curr_region);
+
+ if (limit > bottom) {
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, "
+ "returning it ", task_num, curr_region);
+ return curr_region;
+ } else {
+ tmp_guarantee_CM( limit == bottom,
+ "the region limit should be at bottom" );
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, "
+ "returning NULL", task_num, curr_region);
+ // we return NULL and the caller should try calling
+ // claim_region() again.
+ return NULL;
+ }
+ } else {
+ guarantee( _finger > finger, "the finger should have moved forward" );
+ if (verbose_low())
+ gclog_or_tty->print_cr("[%d] somebody else moved the finger, "
+ "global finger = "PTR_FORMAT", "
+ "our finger = "PTR_FORMAT,
+ task_num, _finger, finger);
+
+ // read it again
+ finger = _finger;
+ }
+ }
+
+ return NULL;
+}
+
+void ConcurrentMark::oops_do(OopClosure* cl) {
+ if (_markStack.size() > 0 && verbose_low())
+ gclog_or_tty->print_cr("[global] scanning the global marking stack, "
+ "size = %d", _markStack.size());
+ // we first iterate over the contents of the mark stack...
+ _markStack.oops_do(cl);
+
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ OopTaskQueue* queue = _task_queues->queue((int)i);
+
+ if (queue->size() > 0 && verbose_low())
+ gclog_or_tty->print_cr("[global] scanning task queue of task %d, "
+ "size = %d", i, queue->size());
+
+ // ...then over the contents of the all the task queues.
+ queue->oops_do(cl);
+ }
+
+ // finally, invalidate any entries that in the region stack that
+ // point into the collection set
+ if (_regionStack.invalidate_entries_into_cset()) {
+ // otherwise, any gray objects copied during the evacuation pause
+ // might not be visited.
+ guarantee( _should_gray_objects, "invariant" );
+ }
+}
+
+void ConcurrentMark::clear_marking_state() {
+ _markStack.setEmpty();
+ _markStack.clear_overflow();
+ _regionStack.setEmpty();
+ _regionStack.clear_overflow();
+ clear_has_overflown();
+ _finger = _heap_start;
+
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ OopTaskQueue* queue = _task_queues->queue(i);
+ queue->set_empty();
+ }
+}
+
+void ConcurrentMark::print_stats() {
+ if (verbose_stats()) {
+ gclog_or_tty->print_cr("---------------------------------------------------------------------");
+ for (size_t i = 0; i < _active_tasks; ++i) {
+ _tasks[i]->print_stats();
+ gclog_or_tty->print_cr("---------------------------------------------------------------------");
+ }
+ }
+}
+
+class CSMarkOopClosure: public OopClosure {
+ friend class CSMarkBitMapClosure;
+
+ G1CollectedHeap* _g1h;
+ CMBitMap* _bm;
+ ConcurrentMark* _cm;
+ oop* _ms;
+ jint* _array_ind_stack;
+ int _ms_size;
+ int _ms_ind;
+ int _array_increment;
+
+ bool push(oop obj, int arr_ind = 0) {
+ if (_ms_ind == _ms_size) {
+ gclog_or_tty->print_cr("Mark stack is full.");
+ return false;
+ }
+ _ms[_ms_ind] = obj;
+ if (obj->is_objArray()) _array_ind_stack[_ms_ind] = arr_ind;
+ _ms_ind++;
+ return true;
+ }
+
+ oop pop() {
+ if (_ms_ind == 0) return NULL;
+ else {
+ _ms_ind--;
+ return _ms[_ms_ind];
+ }
+ }
+
+ bool drain() {
+ while (_ms_ind > 0) {
+ oop obj = pop();
+ assert(obj != NULL, "Since index was non-zero.");
+ if (obj->is_objArray()) {
+ jint arr_ind = _array_ind_stack[_ms_ind];
+ objArrayOop aobj = objArrayOop(obj);
+ jint len = aobj->length();
+ jint next_arr_ind = arr_ind + _array_increment;
+ if (next_arr_ind < len) {
+ push(obj, next_arr_ind);
+ }
+ // Now process this portion of this one.
+ int lim = MIN2(next_arr_ind, len);
+ assert(!UseCompressedOops, "This needs to be fixed");
+ for (int j = arr_ind; j < lim; j++) {
+ do_oop(aobj->obj_at_addr(j));
+ }
+
+ } else {
+ obj->oop_iterate(this);
+ }
+ if (abort()) return false;
+ }
+ return true;
+ }
+
+public:
+ CSMarkOopClosure(ConcurrentMark* cm, int ms_size) :
+ _g1h(G1CollectedHeap::heap()),
+ _cm(cm),
+ _bm(cm->nextMarkBitMap()),
+ _ms_size(ms_size), _ms_ind(0),
+ _ms(NEW_C_HEAP_ARRAY(oop, ms_size)),
+ _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)),
+ _array_increment(MAX2(ms_size/8, 16))
+ {}
+
+ ~CSMarkOopClosure() {
+ FREE_C_HEAP_ARRAY(oop, _ms);
+ FREE_C_HEAP_ARRAY(jint, _array_ind_stack);
+ }
+
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ void do_oop(oop* p) {
+ oop obj = *p;
+ if (obj == NULL) return;
+ if (obj->is_forwarded()) {
+ // If the object has already been forwarded, we have to make sure
+ // that it's marked. So follow the forwarding pointer. Note that
+ // this does the right thing for self-forwarding pointers in the
+ // evacuation failure case.
+ obj = obj->forwardee();
+ }
+ HeapRegion* hr = _g1h->heap_region_containing(obj);
+ if (hr != NULL) {
+ if (hr->in_collection_set()) {
+ if (_g1h->is_obj_ill(obj)) {
+ _bm->mark((HeapWord*)obj);
+ if (!push(obj)) {
+ gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed.");
+ set_abort();
+ }
+ }
+ } else {
+ // Outside the collection set; we need to gray it
+ _cm->deal_with_reference(obj);
+ }
+ }
+ }
+};
+
+class CSMarkBitMapClosure: public BitMapClosure {
+ G1CollectedHeap* _g1h;
+ CMBitMap* _bitMap;
+ ConcurrentMark* _cm;
+ CSMarkOopClosure _oop_cl;
+public:
+ CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) :
+ _g1h(G1CollectedHeap::heap()),
+ _bitMap(cm->nextMarkBitMap()),
+ _oop_cl(cm, ms_size)
+ {}
+
+ ~CSMarkBitMapClosure() {}
+
+ bool do_bit(size_t offset) {
+ // convert offset into a HeapWord*
+ HeapWord* addr = _bitMap->offsetToHeapWord(offset);
+ assert(_bitMap->endWord() && addr < _bitMap->endWord(),
+ "address out of range");
+ assert(_bitMap->isMarked(addr), "tautology");
+ oop obj = oop(addr);
+ if (!obj->is_forwarded()) {
+ if (!_oop_cl.push(obj)) return false;
+ if (!_oop_cl.drain()) return false;
+ }
+ // Otherwise...
+ return true;
+ }
+};
+
+
+class CompleteMarkingInCSHRClosure: public HeapRegionClosure {
+ CMBitMap* _bm;
+ CSMarkBitMapClosure _bit_cl;
+ enum SomePrivateConstants {
+ MSSize = 1000
+ };
+ bool _completed;
+public:
+ CompleteMarkingInCSHRClosure(ConcurrentMark* cm) :
+ _bm(cm->nextMarkBitMap()),
+ _bit_cl(cm, MSSize),
+ _completed(true)
+ {}
+
+ ~CompleteMarkingInCSHRClosure() {}
+
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->evacuation_failed()) {
+ MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start());
+ if (!mr.is_empty()) {
+ if (!_bm->iterate(&_bit_cl, mr)) {
+ _completed = false;
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ bool completed() { return _completed; }
+};
+
+class ClearMarksInHRClosure: public HeapRegionClosure {
+ CMBitMap* _bm;
+public:
+ ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { }
+
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->used_region().is_empty() && !r->evacuation_failed()) {
+ MemRegion usedMR = r->used_region();
+ _bm->clearRange(r->used_region());
+ }
+ return false;
+ }
+};
+
+void ConcurrentMark::complete_marking_in_collection_set() {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+ if (!g1h->mark_in_progress()) {
+ g1h->g1_policy()->record_mark_closure_time(0.0);
+ return;
+ }
+
+ int i = 1;
+ double start = os::elapsedTime();
+ while (true) {
+ i++;
+ CompleteMarkingInCSHRClosure cmplt(this);
+ g1h->collection_set_iterate(&cmplt);
+ if (cmplt.completed()) break;
+ }
+ double end_time = os::elapsedTime();
+ double elapsed_time_ms = (end_time - start) * 1000.0;
+ g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms);
+ if (PrintGCDetails) {
+ gclog_or_tty->print_cr("Mark closure took %5.2f ms.", elapsed_time_ms);
+ }
+
+ ClearMarksInHRClosure clr(nextMarkBitMap());
+ g1h->collection_set_iterate(&clr);
+}
+
+// The next two methods deal with the following optimisation. Some
+// objects are gray by being marked and located above the finger. If
+// they are copied, during an evacuation pause, below the finger then
+// the need to be pushed on the stack. The observation is that, if
+// there are no regions in the collection set located above the
+// finger, then the above cannot happen, hence we do not need to
+// explicitly gray any objects when copying them to below the
+// finger. The global stack will be scanned to ensure that, if it
+// points to objects being copied, it will update their
+// location. There is a tricky situation with the gray objects in
+// region stack that are being coped, however. See the comment in
+// newCSet().
+
+void ConcurrentMark::newCSet() {
+ if (!concurrent_marking_in_progress())
+ // nothing to do if marking is not in progress
+ return;
+
+ // find what the lowest finger is among the global and local fingers
+ _min_finger = _finger;
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ CMTask* task = _tasks[i];
+ HeapWord* task_finger = task->finger();
+ if (task_finger != NULL && task_finger < _min_finger)
+ _min_finger = task_finger;
+ }
+
+ _should_gray_objects = false;
+
+ // This fixes a very subtle and fustrating bug. It might be the case
+ // that, during en evacuation pause, heap regions that contain
+ // objects that are gray (by being in regions contained in the
+ // region stack) are included in the collection set. Since such gray
+ // objects will be moved, and because it's not easy to redirect
+ // region stack entries to point to a new location (because objects
+ // in one region might be scattered to multiple regions after they
+ // are copied), one option is to ensure that all marked objects
+ // copied during a pause are pushed on the stack. Notice, however,
+ // that this problem can only happen when the region stack is not
+ // empty during an evacuation pause. So, we make the fix a bit less
+ // conservative and ensure that regions are pushed on the stack,
+ // irrespective whether all collection set regions are below the
+ // finger, if the region stack is not empty. This is expected to be
+ // a rare case, so I don't think it's necessary to be smarted about it.
+ if (!region_stack_empty())
+ _should_gray_objects = true;
+}
+
+void ConcurrentMark::registerCSetRegion(HeapRegion* hr) {
+ if (!concurrent_marking_in_progress())
+ return;
+
+ HeapWord* region_end = hr->end();
+ if (region_end > _min_finger)
+ _should_gray_objects = true;
+}
+
+void ConcurrentMark::disable_co_trackers() {
+ if (has_aborted()) {
+ if (_cleanup_co_tracker.enabled())
+ _cleanup_co_tracker.disable();
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ CMTask* task = _tasks[i];
+ if (task->co_tracker_enabled())
+ task->disable_co_tracker();
+ }
+ } else {
+ guarantee( !_cleanup_co_tracker.enabled(), "invariant" );
+ for (int i = 0; i < (int)_max_task_num; ++i) {
+ CMTask* task = _tasks[i];
+ guarantee( !task->co_tracker_enabled(), "invariant" );
+ }
+ }
+}
+
+// abandon current marking iteration due to a Full GC
+void ConcurrentMark::abort() {
+ // If we're not marking, nothing to do.
+ if (!G1ConcMark) return;
+
+ // Clear all marks to force marking thread to do nothing
+ _nextMarkBitMap->clearAll();
+ // Empty mark stack
+ clear_marking_state();
+ for (int i = 0; i < (int)_max_task_num; ++i)
+ _tasks[i]->clear_region_fields();
+ _has_aborted = true;
+
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ satb_mq_set.abandon_partial_marking();
+ satb_mq_set.set_active_all_threads(false);
+}
+
+static void print_ms_time_info(const char* prefix, const char* name,
+ NumberSeq& ns) {
+ gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).",
+ prefix, ns.num(), name, ns.sum()/1000.0, ns.avg());
+ if (ns.num() > 0) {
+ gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]",
+ prefix, ns.sd(), ns.maximum());
+ }
+}
+
+void ConcurrentMark::print_summary_info() {
+ gclog_or_tty->print_cr(" Concurrent marking:");
+ print_ms_time_info(" ", "init marks", _init_times);
+ print_ms_time_info(" ", "remarks", _remark_times);
+ {
+ print_ms_time_info(" ", "final marks", _remark_mark_times);
+ print_ms_time_info(" ", "weak refs", _remark_weak_ref_times);
+
+ }
+ print_ms_time_info(" ", "cleanups", _cleanup_times);
+ gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).",
+ _total_counting_time,
+ (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 /
+ (double)_cleanup_times.num()
+ : 0.0));
+ if (G1ScrubRemSets) {
+ gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).",
+ _total_rs_scrub_time,
+ (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 /
+ (double)_cleanup_times.num()
+ : 0.0));
+ }
+ gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.",
+ (_init_times.sum() + _remark_times.sum() +
+ _cleanup_times.sum())/1000.0);
+ gclog_or_tty->print_cr(" Total concurrent time = %8.2f s "
+ "(%8.2f s marking, %8.2f s counting).",
+ cmThread()->vtime_accum(),
+ cmThread()->vtime_mark_accum(),
+ cmThread()->vtime_count_accum());
+}
+
+// Closures
+// XXX: there seems to be a lot of code duplication here;
+// should refactor and consolidate the shared code.
+
+// This closure is used to mark refs into the CMS generation in
+// the CMS bit map. Called at the first checkpoint.
+
+// We take a break if someone is trying to stop the world.
+bool ConcurrentMark::do_yield_check(int worker_i) {
+ if (should_yield()) {
+ if (worker_i == 0)
+ _g1h->g1_policy()->record_concurrent_pause();
+ cmThread()->yield();
+ if (worker_i == 0)
+ _g1h->g1_policy()->record_concurrent_pause_end();
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool ConcurrentMark::should_yield() {
+ return cmThread()->should_yield();
+}
+
+bool ConcurrentMark::containing_card_is_marked(void* p) {
+ size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1);
+ return _card_bm.at(offset >> CardTableModRefBS::card_shift);
+}
+
+bool ConcurrentMark::containing_cards_are_marked(void* start,
+ void* last) {
+ return
+ containing_card_is_marked(start) &&
+ containing_card_is_marked(last);
+}
+
+#ifndef PRODUCT
+// for debugging purposes
+void ConcurrentMark::print_finger() {
+ gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT,
+ _heap_start, _heap_end, _finger);
+ for (int i = 0; i < (int) _max_task_num; ++i) {
+ gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger());
+ }
+ gclog_or_tty->print_cr("");
+}
+#endif
+
+// Closure for iteration over bitmaps
+class CMBitMapClosure : public BitMapClosure {
+private:
+ // the bitmap that is being iterated over
+ CMBitMap* _nextMarkBitMap;
+ ConcurrentMark* _cm;
+ CMTask* _task;
+ // true if we're scanning a heap region claimed by the task (so that
+ // we move the finger along), false if we're not, i.e. currently when
+ // scanning a heap region popped from the region stack (so that we
+ // do not move the task finger along; it'd be a mistake if we did so).
+ bool _scanning_heap_region;
+
+public:
+ CMBitMapClosure(CMTask *task,
+ ConcurrentMark* cm,
+ CMBitMap* nextMarkBitMap)
+ : _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { }
+
+ void set_scanning_heap_region(bool scanning_heap_region) {
+ _scanning_heap_region = scanning_heap_region;
+ }
+
+ bool do_bit(size_t offset) {
+ HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset);
+ tmp_guarantee_CM( _nextMarkBitMap->isMarked(addr), "invariant" );
+ tmp_guarantee_CM( addr < _cm->finger(), "invariant" );
+
+ if (_scanning_heap_region) {
+ statsOnly( _task->increase_objs_found_on_bitmap() );
+ tmp_guarantee_CM( addr >= _task->finger(), "invariant" );
+ // We move that task's local finger along.
+ _task->move_finger_to(addr);
+ } else {
+ // We move the task's region finger along.
+ _task->move_region_finger_to(addr);
+ }
+
+ _task->scan_object(oop(addr));
+ // we only partially drain the local queue and global stack
+ _task->drain_local_queue(true);
+ _task->drain_global_stack(true);
+
+ // if the has_aborted flag has been raised, we need to bail out of
+ // the iteration
+ return !_task->has_aborted();
+ }
+};
+
+// Closure for iterating over objects, currently only used for
+// processing SATB buffers.
+class CMObjectClosure : public ObjectClosure {
+private:
+ CMTask* _task;
+
+public:
+ void do_object(oop obj) {
+ _task->deal_with_reference(obj);
+ }
+
+ CMObjectClosure(CMTask* task) : _task(task) { }
+};
+
+// Closure for iterating over object fields
+class CMOopClosure : public OopClosure {
+private:
+ G1CollectedHeap* _g1h;
+ ConcurrentMark* _cm;
+ CMTask* _task;
+
+public:
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ void do_oop(oop* p) {
+ tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant" );
+
+ oop obj = *p;
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] we're looking at location "
+ "*"PTR_FORMAT" = "PTR_FORMAT,
+ _task->task_id(), p, (void*) obj);
+ _task->deal_with_reference(obj);
+ }
+
+ CMOopClosure(G1CollectedHeap* g1h,
+ ConcurrentMark* cm,
+ CMTask* task)
+ : _g1h(g1h), _cm(cm), _task(task) { }
+};
+
+void CMTask::setup_for_region(HeapRegion* hr) {
+ tmp_guarantee_CM( hr != NULL && !hr->continuesHumongous(),
+ "claim_region() should have filtered out continues humongous regions" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT,
+ _task_id, hr);
+
+ _curr_region = hr;
+ _finger = hr->bottom();
+ update_region_limit();
+}
+
+void CMTask::update_region_limit() {
+ HeapRegion* hr = _curr_region;
+ HeapWord* bottom = hr->bottom();
+ HeapWord* limit = hr->next_top_at_mark_start();
+
+ if (limit == bottom) {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] found an empty region "
+ "["PTR_FORMAT", "PTR_FORMAT")",
+ _task_id, bottom, limit);
+ // The region was collected underneath our feet.
+ // We set the finger to bottom to ensure that the bitmap
+ // iteration that will follow this will not do anything.
+ // (this is not a condition that holds when we set the region up,
+ // as the region is not supposed to be empty in the first place)
+ _finger = bottom;
+ } else if (limit >= _region_limit) {
+ tmp_guarantee_CM( limit >= _finger, "peace of mind" );
+ } else {
+ tmp_guarantee_CM( limit < _region_limit, "only way to get here" );
+ // This can happen under some pretty unusual circumstances. An
+ // evacuation pause empties the region underneath our feet (NTAMS
+ // at bottom). We then do some allocation in the region (NTAMS
+ // stays at bottom), followed by the region being used as a GC
+ // alloc region (NTAMS will move to top() and the objects
+ // originally below it will be grayed). All objects now marked in
+ // the region are explicitly grayed, if below the global finger,
+ // and we do not need in fact to scan anything else. So, we simply
+ // set _finger to be limit to ensure that the bitmap iteration
+ // doesn't do anything.
+ _finger = limit;
+ }
+
+ _region_limit = limit;
+}
+
+void CMTask::giveup_current_region() {
+ tmp_guarantee_CM( _curr_region != NULL, "invariant" );
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT,
+ _task_id, _curr_region);
+ clear_region_fields();
+}
+
+void CMTask::clear_region_fields() {
+ // Values for these three fields that indicate that we're not
+ // holding on to a region.
+ _curr_region = NULL;
+ _finger = NULL;
+ _region_limit = NULL;
+
+ _region_finger = NULL;
+}
+
+void CMTask::reset(CMBitMap* nextMarkBitMap) {
+ guarantee( nextMarkBitMap != NULL, "invariant" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] resetting", _task_id);
+
+ _nextMarkBitMap = nextMarkBitMap;
+ clear_region_fields();
+
+ _calls = 0;
+ _elapsed_time_ms = 0.0;
+ _termination_time_ms = 0.0;
+ _termination_start_time_ms = 0.0;
+
+#if _MARKING_STATS_
+ _local_pushes = 0;
+ _local_pops = 0;
+ _local_max_size = 0;
+ _objs_scanned = 0;
+ _global_pushes = 0;
+ _global_pops = 0;
+ _global_max_size = 0;
+ _global_transfers_to = 0;
+ _global_transfers_from = 0;
+ _region_stack_pops = 0;
+ _regions_claimed = 0;
+ _objs_found_on_bitmap = 0;
+ _satb_buffers_processed = 0;
+ _steal_attempts = 0;
+ _steals = 0;
+ _aborted = 0;
+ _aborted_overflow = 0;
+ _aborted_cm_aborted = 0;
+ _aborted_yield = 0;
+ _aborted_timed_out = 0;
+ _aborted_satb = 0;
+ _aborted_termination = 0;
+#endif // _MARKING_STATS_
+}
+
+bool CMTask::should_exit_termination() {
+ regular_clock_call();
+ // This is called when we are in the termination protocol. We should
+ // quit if, for some reason, this task wants to abort or the global
+ // stack is not empty (this means that we can get work from it).
+ return !_cm->mark_stack_empty() || has_aborted();
+}
+
+// This determines whether the method below will check both the local
+// and global fingers when determining whether to push on the stack a
+// gray object (value 1) or whether it will only check the global one
+// (value 0). The tradeoffs are that the former will be a bit more
+// accurate and possibly push less on the stack, but it might also be
+// a little bit slower.
+
+#define _CHECK_BOTH_FINGERS_ 1
+
+void CMTask::deal_with_reference(oop obj) {
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT,
+ _task_id, (void*) obj);
+
+ ++_refs_reached;
+
+ HeapWord* objAddr = (HeapWord*) obj;
+ if (_g1h->is_in_g1_reserved(objAddr)) {
+ tmp_guarantee_CM( obj != NULL, "is_in_g1_reserved should ensure this" );
+ HeapRegion* hr = _g1h->heap_region_containing(obj);
+ if (_g1h->is_obj_ill(obj, hr)) {
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked",
+ _task_id, (void*) obj);
+
+ // we need to mark it first
+ if (_nextMarkBitMap->parMark(objAddr)) {
+ // No OrderAccess:store_load() is needed. It is implicit in the
+ // CAS done in parMark(objAddr) above
+ HeapWord* global_finger = _cm->finger();
+
+#if _CHECK_BOTH_FINGERS_
+ // we will check both the local and global fingers
+
+ if (_finger != NULL && objAddr < _finger) {
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), "
+ "pushing it", _task_id, _finger);
+ push(obj);
+ } else if (_curr_region != NULL && objAddr < _region_limit) {
+ // do nothing
+ } else if (objAddr < global_finger) {
+ // Notice that the global finger might be moving forward
+ // concurrently. This is not a problem. In the worst case, we
+ // mark the object while it is above the global finger and, by
+ // the time we read the global finger, it has moved forward
+ // passed this object. In this case, the object will probably
+ // be visited when a task is scanning the region and will also
+ // be pushed on the stack. So, some duplicate work, but no
+ // correctness problems.
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] below the global finger "
+ "("PTR_FORMAT"), pushing it",
+ _task_id, global_finger);
+ push(obj);
+ } else {
+ // do nothing
+ }
+#else // _CHECK_BOTH_FINGERS_
+ // we will only check the global finger
+
+ if (objAddr < global_finger) {
+ // see long comment above
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] below the global finger "
+ "("PTR_FORMAT"), pushing it",
+ _task_id, global_finger);
+ push(obj);
+ }
+#endif // _CHECK_BOTH_FINGERS_
+ }
+ }
+ }
+}
+
+void CMTask::push(oop obj) {
+ HeapWord* objAddr = (HeapWord*) obj;
+ tmp_guarantee_CM( _g1h->is_in_g1_reserved(objAddr), "invariant" );
+ tmp_guarantee_CM( !_g1h->is_obj_ill(obj), "invariant" );
+ tmp_guarantee_CM( _nextMarkBitMap->isMarked(objAddr), "invariant" );
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
+
+ if (!_task_queue->push(obj)) {
+ // The local task queue looks full. We need to push some entries
+ // to the global stack.
+
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] task queue overflow, "
+ "moving entries to the global stack",
+ _task_id);
+ move_entries_to_global_stack();
+
+ // this should succeed since, even if we overflow the global
+ // stack, we should have definitely removed some entries from the
+ // local queue. So, there must be space on it.
+ bool success = _task_queue->push(obj);
+ tmp_guarantee_CM( success, "invariant" );
+ }
+
+ statsOnly( int tmp_size = _task_queue->size();
+ if (tmp_size > _local_max_size)
+ _local_max_size = tmp_size;
+ ++_local_pushes );
+}
+
+void CMTask::reached_limit() {
+ tmp_guarantee_CM( _words_scanned >= _words_scanned_limit ||
+ _refs_reached >= _refs_reached_limit ,
+ "shouldn't have been called otherwise" );
+ regular_clock_call();
+}
+
+void CMTask::regular_clock_call() {
+ if (has_aborted())
+ return;
+
+ // First, we need to recalculate the words scanned and refs reached
+ // limits for the next clock call.
+ recalculate_limits();
+
+ // During the regular clock call we do the following
+
+ // (1) If an overflow has been flagged, then we abort.
+ if (_cm->has_overflown()) {
+ set_has_aborted();
+ return;
+ }
+
+ // If we are not concurrent (i.e. we're doing remark) we don't need
+ // to check anything else. The other steps are only needed during
+ // the concurrent marking phase.
+ if (!concurrent())
+ return;
+
+ // (2) If marking has been aborted for Full GC, then we also abort.
+ if (_cm->has_aborted()) {
+ set_has_aborted();
+ statsOnly( ++_aborted_cm_aborted );
+ return;
+ }
+
+ double curr_time_ms = os::elapsedVTime() * 1000.0;
+
+ // (3) If marking stats are enabled, then we update the step history.
+#if _MARKING_STATS_
+ if (_words_scanned >= _words_scanned_limit)
+ ++_clock_due_to_scanning;
+ if (_refs_reached >= _refs_reached_limit)
+ ++_clock_due_to_marking;
+
+ double last_interval_ms = curr_time_ms - _interval_start_time_ms;
+ _interval_start_time_ms = curr_time_ms;
+ _all_clock_intervals_ms.add(last_interval_ms);
+
+ if (_cm->verbose_medium()) {
+ gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, "
+ "scanned = %d%s, refs reached = %d%s",
+ _task_id, last_interval_ms,
+ _words_scanned,
+ (_words_scanned >= _words_scanned_limit) ? " (*)" : "",
+ _refs_reached,
+ (_refs_reached >= _refs_reached_limit) ? " (*)" : "");
+ }
+#endif // _MARKING_STATS_
+
+ // (4) We check whether we should yield. If we have to, then we abort.
+ if (_cm->should_yield()) {
+ // We should yield. To do this we abort the task. The caller is
+ // responsible for yielding.
+ set_has_aborted();
+ statsOnly( ++_aborted_yield );
+ return;
+ }
+
+ // (5) We check whether we've reached our time quota. If we have,
+ // then we abort.
+ double elapsed_time_ms = curr_time_ms - _start_time_ms;
+ if (elapsed_time_ms > _time_target_ms) {
+ set_has_aborted();
+ _has_aborted_timed_out = true;
+ statsOnly( ++_aborted_timed_out );
+ return;
+ }
+
+ // (6) Finally, we check whether there are enough completed STAB
+ // buffers available for processing. If there are, we abort.
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers",
+ _task_id);
+ // we do need to process SATB buffers, we'll abort and restart
+ // the marking task to do so
+ set_has_aborted();
+ statsOnly( ++_aborted_satb );
+ return;
+ }
+}
+
+void CMTask::recalculate_limits() {
+ _real_words_scanned_limit = _words_scanned + words_scanned_period;
+ _words_scanned_limit = _real_words_scanned_limit;
+
+ _real_refs_reached_limit = _refs_reached + refs_reached_period;
+ _refs_reached_limit = _real_refs_reached_limit;
+}
+
+void CMTask::decrease_limits() {
+ // This is called when we believe that we're going to do an infrequent
+ // operation which will increase the per byte scanned cost (i.e. move
+ // entries to/from the global stack). It basically tries to decrease the
+ // scanning limit so that the clock is called earlier.
+
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] decreasing limits", _task_id);
+
+ _words_scanned_limit = _real_words_scanned_limit -
+ 3 * words_scanned_period / 4;
+ _refs_reached_limit = _real_refs_reached_limit -
+ 3 * refs_reached_period / 4;
+}
+
+void CMTask::move_entries_to_global_stack() {
+ // local array where we'll store the entries that will be popped
+ // from the local queue
+ oop buffer[global_stack_transfer_size];
+
+ int n = 0;
+ oop obj;
+ while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) {
+ buffer[n] = obj;
+ ++n;
+ }
+
+ if (n > 0) {
+ // we popped at least one entry from the local queue
+
+ statsOnly( ++_global_transfers_to; _local_pops += n );
+
+ if (!_cm->mark_stack_push(buffer, n)) {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", _task_id);
+ set_has_aborted();
+ } else {
+ // the transfer was successful
+
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack",
+ _task_id, n);
+ statsOnly( int tmp_size = _cm->mark_stack_size();
+ if (tmp_size > _global_max_size)
+ _global_max_size = tmp_size;
+ _global_pushes += n );
+ }
+ }
+
+ // this operation was quite expensive, so decrease the limits
+ decrease_limits();
+}
+
+void CMTask::get_entries_from_global_stack() {
+ // local array where we'll store the entries that will be popped
+ // from the global stack.
+ oop buffer[global_stack_transfer_size];
+ int n;
+ _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n);
+ tmp_guarantee_CM( n <= global_stack_transfer_size,
+ "we should not pop more than the given limit" );
+ if (n > 0) {
+ // yes, we did actually pop at least one entry
+
+ statsOnly( ++_global_transfers_from; _global_pops += n );
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] popped %d entries from the global stack",
+ _task_id, n);
+ for (int i = 0; i < n; ++i) {
+ bool success = _task_queue->push(buffer[i]);
+ // We only call this when the local queue is empty or under a
+ // given target limit. So, we do not expect this push to fail.
+ tmp_guarantee_CM( success, "invariant" );
+ }
+
+ statsOnly( int tmp_size = _task_queue->size();
+ if (tmp_size > _local_max_size)
+ _local_max_size = tmp_size;
+ _local_pushes += n );
+ }
+
+ // this operation was quite expensive, so decrease the limits
+ decrease_limits();
+}
+
+void CMTask::drain_local_queue(bool partially) {
+ if (has_aborted())
+ return;
+
+ // Decide what the target size is, depending whether we're going to
+ // drain it partially (so that other tasks can steal if they run out
+ // of things to do) or totally (at the very end).
+ size_t target_size;
+ if (partially)
+ target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize);
+ else
+ target_size = 0;
+
+ if (_task_queue->size() > target_size) {
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] draining local queue, target size = %d",
+ _task_id, target_size);
+
+ oop obj;
+ bool ret = _task_queue->pop_local(obj);
+ while (ret) {
+ statsOnly( ++_local_pops );
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id,
+ (void*) obj);
+
+ tmp_guarantee_CM( _g1h->is_in_g1_reserved((HeapWord*) obj),
+ "invariant" );
+
+ scan_object(obj);
+
+ if (_task_queue->size() <= target_size || has_aborted())
+ ret = false;
+ else
+ ret = _task_queue->pop_local(obj);
+ }
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] drained local queue, size = %d",
+ _task_id, _task_queue->size());
+ }
+}
+
+void CMTask::drain_global_stack(bool partially) {
+ if (has_aborted())
+ return;
+
+ // We have a policy to drain the local queue before we attempt to
+ // drain the global stack.
+ tmp_guarantee_CM( partially || _task_queue->size() == 0, "invariant" );
+
+ // Decide what the target size is, depending whether we're going to
+ // drain it partially (so that other tasks can steal if they run out
+ // of things to do) or totally (at the very end). Notice that,
+ // because we move entries from the global stack in chunks or
+ // because another task might be doing the same, we might in fact
+ // drop below the target. But, this is not a problem.
+ size_t target_size;
+ if (partially)
+ target_size = _cm->partial_mark_stack_size_target();
+ else
+ target_size = 0;
+
+ if (_cm->mark_stack_size() > target_size) {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] draining global_stack, target size %d",
+ _task_id, target_size);
+
+ while (!has_aborted() && _cm->mark_stack_size() > target_size) {
+ get_entries_from_global_stack();
+ drain_local_queue(partially);
+ }
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] drained global stack, size = %d",
+ _task_id, _cm->mark_stack_size());
+ }
+}
+
+// SATB Queue has several assumptions on whether to call the par or
+// non-par versions of the methods. this is why some of the code is
+// replicated. We should really get rid of the single-threaded version
+// of the code to simplify things.
+void CMTask::drain_satb_buffers() {
+ if (has_aborted())
+ return;
+
+ // We set this so that the regular clock knows that we're in the
+ // middle of draining buffers and doesn't set the abort flag when it
+ // notices that SATB buffers are available for draining. It'd be
+ // very counter productive if it did that. :-)
+ _draining_satb_buffers = true;
+
+ CMObjectClosure oc(this);
+ SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
+ if (ParallelGCThreads > 0)
+ satb_mq_set.set_par_closure(_task_id, &oc);
+ else
+ satb_mq_set.set_closure(&oc);
+
+ // This keeps claiming and applying the closure to completed buffers
+ // until we run out of buffers or we need to abort.
+ if (ParallelGCThreads > 0) {
+ while (!has_aborted() &&
+ satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) {
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
+ statsOnly( ++_satb_buffers_processed );
+ regular_clock_call();
+ }
+ } else {
+ while (!has_aborted() &&
+ satb_mq_set.apply_closure_to_completed_buffer()) {
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id);
+ statsOnly( ++_satb_buffers_processed );
+ regular_clock_call();
+ }
+ }
+
+ if (!concurrent() && !has_aborted()) {
+ // We should only do this during remark.
+ if (ParallelGCThreads > 0)
+ satb_mq_set.par_iterate_closure_all_threads(_task_id);
+ else
+ satb_mq_set.iterate_closure_all_threads();
+ }
+
+ _draining_satb_buffers = false;
+
+ tmp_guarantee_CM( has_aborted() ||
+ concurrent() ||
+ satb_mq_set.completed_buffers_num() == 0, "invariant" );
+
+ if (ParallelGCThreads > 0)
+ satb_mq_set.set_par_closure(_task_id, NULL);
+ else
+ satb_mq_set.set_closure(NULL);
+
+ // again, this was a potentially expensive operation, decrease the
+ // limits to get the regular clock call early
+ decrease_limits();
+}
+
+void CMTask::drain_region_stack(BitMapClosure* bc) {
+ if (has_aborted())
+ return;
+
+ tmp_guarantee_CM( _region_finger == NULL,
+ "it should be NULL when we're not scanning a region" );
+
+ if (!_cm->region_stack_empty()) {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] draining region stack, size = %d",
+ _task_id, _cm->region_stack_size());
+
+ MemRegion mr = _cm->region_stack_pop();
+ // it returns MemRegion() if the pop fails
+ statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
+
+ while (mr.start() != NULL) {
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] we are scanning region "
+ "["PTR_FORMAT", "PTR_FORMAT")",
+ _task_id, mr.start(), mr.end());
+ tmp_guarantee_CM( mr.end() <= _cm->finger(),
+ "otherwise the region shouldn't be on the stack" );
+ assert(!mr.is_empty(), "Only non-empty regions live on the region stack");
+ if (_nextMarkBitMap->iterate(bc, mr)) {
+ tmp_guarantee_CM( !has_aborted(),
+ "cannot abort the task without aborting the bitmap iteration" );
+
+ // We finished iterating over the region without aborting.
+ regular_clock_call();
+ if (has_aborted())
+ mr = MemRegion();
+ else {
+ mr = _cm->region_stack_pop();
+ // it returns MemRegion() if the pop fails
+ statsOnly(if (mr.start() != NULL) ++_region_stack_pops );
+ }
+ } else {
+ guarantee( has_aborted(), "currently the only way to do so" );
+
+ // The only way to abort the bitmap iteration is to return
+ // false from the do_bit() method. However, inside the
+ // do_bit() method we move the _region_finger to point to the
+ // object currently being looked at. So, if we bail out, we
+ // have definitely set _region_finger to something non-null.
+ guarantee( _region_finger != NULL, "invariant" );
+
+ // The iteration was actually aborted. So now _region_finger
+ // points to the address of the object we last scanned. If we
+ // leave it there, when we restart this task, we will rescan
+ // the object. It is easy to avoid this. We move the finger by
+ // enough to point to the next possible object header (the
+ // bitmap knows by how much we need to move it as it knows its
+ // granularity).
+ MemRegion newRegion =
+ MemRegion(_nextMarkBitMap->nextWord(_region_finger), mr.end());
+
+ if (!newRegion.is_empty()) {
+ if (_cm->verbose_low()) {
+ gclog_or_tty->print_cr("[%d] pushing unscanned region"
+ "[" PTR_FORMAT "," PTR_FORMAT ") on region stack",
+ _task_id,
+ newRegion.start(), newRegion.end());
+ }
+ // Now push the part of the region we didn't scan on the
+ // region stack to make sure a task scans it later.
+ _cm->region_stack_push(newRegion);
+ }
+ // break from while
+ mr = MemRegion();
+ }
+ _region_finger = NULL;
+ }
+
+ // We only push regions on the region stack during evacuation
+ // pauses. So if we come out the above iteration because we region
+ // stack is empty, it will remain empty until the next yield
+ // point. So, the guarantee below is safe.
+ guarantee( has_aborted() || _cm->region_stack_empty(),
+ "only way to exit the loop" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] drained region stack, size = %d",
+ _task_id, _cm->region_stack_size());
+ }
+}
+
+void CMTask::print_stats() {
+ gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d",
+ _task_id, _calls);
+ gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms",
+ _elapsed_time_ms, _termination_time_ms);
+ gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
+ _step_times_ms.num(), _step_times_ms.avg(),
+ _step_times_ms.sd());
+ gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
+ _step_times_ms.maximum(), _step_times_ms.sum());
+
+#if _MARKING_STATS_
+ gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms",
+ _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(),
+ _all_clock_intervals_ms.sd());
+ gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms",
+ _all_clock_intervals_ms.maximum(),
+ _all_clock_intervals_ms.sum());
+ gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d",
+ _clock_due_to_scanning, _clock_due_to_marking);
+ gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d",
+ _objs_scanned, _objs_found_on_bitmap);
+ gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d",
+ _local_pushes, _local_pops, _local_max_size);
+ gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d",
+ _global_pushes, _global_pops, _global_max_size);
+ gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d",
+ _global_transfers_to,_global_transfers_from);
+ gclog_or_tty->print_cr(" Regions: claimed = %d, Region Stack: pops = %d",
+ _regions_claimed, _region_stack_pops);
+ gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed);
+ gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d",
+ _steal_attempts, _steals);
+ gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted);
+ gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d",
+ _aborted_overflow, _aborted_cm_aborted, _aborted_yield);
+ gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d",
+ _aborted_timed_out, _aborted_satb, _aborted_termination);
+#endif // _MARKING_STATS_
+}
+
+/*****************************************************************************
+
+ The do_marking_step(time_target_ms) method is the building block
+ of the parallel marking framework. It can be called in parallel
+ with other invocations of do_marking_step() on different tasks
+ (but only one per task, obviously) and concurrently with the
+ mutator threads, or during remark, hence it eliminates the need
+ for two versions of the code. When called during remark, it will
+ pick up from where the task left off during the concurrent marking
+ phase. Interestingly, tasks are also claimable during evacuation
+ pauses too, since do_marking_step() ensures that it aborts before
+ it needs to yield.
+
+ The data structures that is uses to do marking work are the
+ following:
+
+ (1) Marking Bitmap. If there are gray objects that appear only
+ on the bitmap (this happens either when dealing with an overflow
+ or when the initial marking phase has simply marked the roots
+ and didn't push them on the stack), then tasks claim heap
+ regions whose bitmap they then scan to find gray objects. A
+ global finger indicates where the end of the last claimed region
+ is. A local finger indicates how far into the region a task has
+ scanned. The two fingers are used to determine how to gray an
+ object (i.e. whether simply marking it is OK, as it will be
+ visited by a task in the future, or whether it needs to be also
+ pushed on a stack).
+
+ (2) Local Queue. The local queue of the task which is accessed
+ reasonably efficiently by the task. Other tasks can steal from
+ it when they run out of work. Throughout the marking phase, a
+ task attempts to keep its local queue short but not totally
+ empty, so that entries are available for stealing by other
+ tasks. Only when there is no more work, a task will totally
+ drain its local queue.
+
+ (3) Global Mark Stack. This handles local queue overflow. During
+ marking only sets of entries are moved between it and the local
+ queues, as access to it requires a mutex and more fine-grain
+ interaction with it which might cause contention. If it
+ overflows, then the marking phase should restart and iterate
+ over the bitmap to identify gray objects. Throughout the marking
+ phase, tasks attempt to keep the global mark stack at a small
+ length but not totally empty, so that entries are available for
+ popping by other tasks. Only when there is no more work, tasks
+ will totally drain the global mark stack.
+
+ (4) Global Region Stack. Entries on it correspond to areas of
+ the bitmap that need to be scanned since they contain gray
+ objects. Pushes on the region stack only happen during
+ evacuation pauses and typically correspond to areas covered by
+ GC LABS. If it overflows, then the marking phase should restart
+ and iterate over the bitmap to identify gray objects. Tasks will
+ try to totally drain the region stack as soon as possible.
+
+ (5) SATB Buffer Queue. This is where completed SATB buffers are
+ made available. Buffers are regularly removed from this queue
+ and scanned for roots, so that the queue doesn't get too
+ long. During remark, all completed buffers are processed, as
+ well as the filled in parts of any uncompleted buffers.
+
+ The do_marking_step() method tries to abort when the time target
+ has been reached. There are a few other cases when the
+ do_marking_step() method also aborts:
+
+ (1) When the marking phase has been aborted (after a Full GC).
+
+ (2) When a global overflow (either on the global stack or the
+ region stack) has been triggered. Before the task aborts, it
+ will actually sync up with the other tasks to ensure that all
+ the marking data structures (local queues, stacks, fingers etc.)
+ are re-initialised so that when do_marking_step() completes,
+ the marking phase can immediately restart.
+
+ (3) When enough completed SATB buffers are available. The
+ do_marking_step() method only tries to drain SATB buffers right
+ at the beginning. So, if enough buffers are available, the
+ marking step aborts and the SATB buffers are processed at
+ the beginning of the next invocation.
+
+ (4) To yield. when we have to yield then we abort and yield
+ right at the end of do_marking_step(). This saves us from a lot
+ of hassle as, by yielding we might allow a Full GC. If this
+ happens then objects will be compacted underneath our feet, the
+ heap might shrink, etc. We save checking for this by just
+ aborting and doing the yield right at the end.
+
+ From the above it follows that the do_marking_step() method should
+ be called in a loop (or, otherwise, regularly) until it completes.
+
+ If a marking step completes without its has_aborted() flag being
+ true, it means it has completed the current marking phase (and
+ also all other marking tasks have done so and have all synced up).
+
+ A method called regular_clock_call() is invoked "regularly" (in
+ sub ms intervals) throughout marking. It is this clock method that
+ checks all the abort conditions which were mentioned above and
+ decides when the task should abort. A work-based scheme is used to
+ trigger this clock method: when the number of object words the
+ marking phase has scanned or the number of references the marking
+ phase has visited reach a given limit. Additional invocations to
+ the method clock have been planted in a few other strategic places
+ too. The initial reason for the clock method was to avoid calling
+ vtime too regularly, as it is quite expensive. So, once it was in
+ place, it was natural to piggy-back all the other conditions on it
+ too and not constantly check them throughout the code.
+
+ *****************************************************************************/
+
+void CMTask::do_marking_step(double time_target_ms) {
+ guarantee( time_target_ms >= 1.0, "minimum granularity is 1ms" );
+ guarantee( concurrent() == _cm->concurrent(), "they should be the same" );
+
+ guarantee( concurrent() || _cm->region_stack_empty(),
+ "the region stack should have been cleared before remark" );
+ guarantee( _region_finger == NULL,
+ "this should be non-null only when a region is being scanned" );
+
+ G1CollectorPolicy* g1_policy = _g1h->g1_policy();
+ guarantee( _task_queues != NULL, "invariant" );
+ guarantee( _task_queue != NULL, "invariant" );
+ guarantee( _task_queues->queue(_task_id) == _task_queue, "invariant" );
+
+ guarantee( !_claimed,
+ "only one thread should claim this task at any one time" );
+
+ // OK, this doesn't safeguard again all possible scenarios, as it is
+ // possible for two threads to set the _claimed flag at the same
+ // time. But it is only for debugging purposes anyway and it will
+ // catch most problems.
+ _claimed = true;
+
+ _start_time_ms = os::elapsedVTime() * 1000.0;
+ statsOnly( _interval_start_time_ms = _start_time_ms );
+
+ double diff_prediction_ms =
+ g1_policy->get_new_prediction(&_marking_step_diffs_ms);
+ _time_target_ms = time_target_ms - diff_prediction_ms;
+
+ // set up the variables that are used in the work-based scheme to
+ // call the regular clock method
+ _words_scanned = 0;
+ _refs_reached = 0;
+ recalculate_limits();
+
+ // clear all flags
+ clear_has_aborted();
+ _has_aborted_timed_out = false;
+ _draining_satb_buffers = false;
+
+ ++_calls;
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, "
+ "target = %1.2lfms >>>>>>>>>>",
+ _task_id, _calls, _time_target_ms);
+
+ // Set up the bitmap and oop closures. Anything that uses them is
+ // eventually called from this method, so it is OK to allocate these
+ // statically.
+ CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap);
+ CMOopClosure oop_closure(_g1h, _cm, this);
+ set_oop_closure(&oop_closure);
+
+ if (_cm->has_overflown()) {
+ // This can happen if the region stack or the mark stack overflows
+ // during a GC pause and this task, after a yield point,
+ // restarts. We have to abort as we need to get into the overflow
+ // protocol which happens right at the end of this task.
+ set_has_aborted();
+ }
+
+ // First drain any available SATB buffers. After this, we will not
+ // look at SATB buffers before the next invocation of this method.
+ // If enough completed SATB buffers are queued up, the regular clock
+ // will abort this task so that it restarts.
+ drain_satb_buffers();
+ // ...then partially drain the local queue and the global stack
+ drain_local_queue(true);
+ drain_global_stack(true);
+
+ // Then totally drain the region stack. We will not look at
+ // it again before the next invocation of this method. Entries on
+ // the region stack are only added during evacuation pauses, for
+ // which we have to yield. When we do, we abort the task anyway so
+ // it will look at the region stack again when it restarts.
+ bitmap_closure.set_scanning_heap_region(false);
+ drain_region_stack(&bitmap_closure);
+ // ...then partially drain the local queue and the global stack
+ drain_local_queue(true);
+ drain_global_stack(true);
+
+ do {
+ if (!has_aborted() && _curr_region != NULL) {
+ // This means that we're already holding on to a region.
+ tmp_guarantee_CM( _finger != NULL,
+ "if region is not NULL, then the finger "
+ "should not be NULL either" );
+
+ // We might have restarted this task after an evacuation pause
+ // which might have evacuated the region we're holding on to
+ // underneath our feet. Let's read its limit again to make sure
+ // that we do not iterate over a region of the heap that
+ // contains garbage (update_region_limit() will also move
+ // _finger to the start of the region if it is found empty).
+ update_region_limit();
+ // We will start from _finger not from the start of the region,
+ // as we might be restarting this task after aborting half-way
+ // through scanning this region. In this case, _finger points to
+ // the address where we last found a marked object. If this is a
+ // fresh region, _finger points to start().
+ MemRegion mr = MemRegion(_finger, _region_limit);
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] we're scanning part "
+ "["PTR_FORMAT", "PTR_FORMAT") "
+ "of region "PTR_FORMAT,
+ _task_id, _finger, _region_limit, _curr_region);
+
+ // Let's iterate over the bitmap of the part of the
+ // region that is left.
+ bitmap_closure.set_scanning_heap_region(true);
+ if (mr.is_empty() ||
+ _nextMarkBitMap->iterate(&bitmap_closure, mr)) {
+ // We successfully completed iterating over the region. Now,
+ // let's give up the region.
+ giveup_current_region();
+ regular_clock_call();
+ } else {
+ guarantee( has_aborted(), "currently the only way to do so" );
+ // The only way to abort the bitmap iteration is to return
+ // false from the do_bit() method. However, inside the
+ // do_bit() method we move the _finger to point to the
+ // object currently being looked at. So, if we bail out, we
+ // have definitely set _finger to something non-null.
+ guarantee( _finger != NULL, "invariant" );
+
+ // Region iteration was actually aborted. So now _finger
+ // points to the address of the object we last scanned. If we
+ // leave it there, when we restart this task, we will rescan
+ // the object. It is easy to avoid this. We move the finger by
+ // enough to point to the next possible object header (the
+ // bitmap knows by how much we need to move it as it knows its
+ // granularity).
+ move_finger_to(_nextMarkBitMap->nextWord(_finger));
+ }
+ }
+ // At this point we have either completed iterating over the
+ // region we were holding on to, or we have aborted.
+
+ // We then partially drain the local queue and the global stack.
+ // (Do we really need this?)
+ drain_local_queue(true);
+ drain_global_stack(true);
+
+ // Read the note on the claim_region() method on why it might
+ // return NULL with potentially more regions available for
+ // claiming and why we have to check out_of_regions() to determine
+ // whether we're done or not.
+ while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) {
+ // We are going to try to claim a new region. We should have
+ // given up on the previous one.
+ tmp_guarantee_CM( _curr_region == NULL &&
+ _finger == NULL &&
+ _region_limit == NULL, "invariant" );
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id);
+ HeapRegion* claimed_region = _cm->claim_region(_task_id);
+ if (claimed_region != NULL) {
+ // Yes, we managed to claim one
+ statsOnly( ++_regions_claimed );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] we successfully claimed "
+ "region "PTR_FORMAT,
+ _task_id, claimed_region);
+
+ setup_for_region(claimed_region);
+ tmp_guarantee_CM( _curr_region == claimed_region, "invariant" );
+ }
+ // It is important to call the regular clock here. It might take
+ // a while to claim a region if, for example, we hit a large
+ // block of empty regions. So we need to call the regular clock
+ // method once round the loop to make sure it's called
+ // frequently enough.
+ regular_clock_call();
+ }
+
+ if (!has_aborted() && _curr_region == NULL) {
+ tmp_guarantee_CM( _cm->out_of_regions(),
+ "at this point we should be out of regions" );
+ }
+ } while ( _curr_region != NULL && !has_aborted());
+
+ if (!has_aborted()) {
+ // We cannot check whether the global stack is empty, since other
+ // tasks might be pushing objects to it concurrently. We also cannot
+ // check if the region stack is empty because if a thread is aborting
+ // it can push a partially done region back.
+ tmp_guarantee_CM( _cm->out_of_regions(),
+ "at this point we should be out of regions" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] all regions claimed", _task_id);
+
+ // Try to reduce the number of available SATB buffers so that
+ // remark has less work to do.
+ drain_satb_buffers();
+ }
+
+ // Since we've done everything else, we can now totally drain the
+ // local queue and global stack.
+ drain_local_queue(false);
+ drain_global_stack(false);
+
+ // Attempt at work stealing from other task's queues.
+ if (!has_aborted()) {
+ // We have not aborted. This means that we have finished all that
+ // we could. Let's try to do some stealing...
+
+ // We cannot check whether the global stack is empty, since other
+ // tasks might be pushing objects to it concurrently. We also cannot
+ // check if the region stack is empty because if a thread is aborting
+ // it can push a partially done region back.
+ guarantee( _cm->out_of_regions() &&
+ _task_queue->size() == 0, "only way to reach here" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] starting to steal", _task_id);
+
+ while (!has_aborted()) {
+ oop obj;
+ statsOnly( ++_steal_attempts );
+
+ if (_cm->try_stealing(_task_id, &_hash_seed, obj)) {
+ if (_cm->verbose_medium())
+ gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully",
+ _task_id, (void*) obj);
+
+ statsOnly( ++_steals );
+
+ tmp_guarantee_CM( _nextMarkBitMap->isMarked((HeapWord*) obj),
+ "any stolen object should be marked" );
+ scan_object(obj);
+
+ // And since we're towards the end, let's totally drain the
+ // local queue and global stack.
+ drain_local_queue(false);
+ drain_global_stack(false);
+ } else {
+ break;
+ }
+ }
+ }
+
+ // We still haven't aborted. Now, let's try to get into the
+ // termination protocol.
+ if (!has_aborted()) {
+ // We cannot check whether the global stack is empty, since other
+ // tasks might be concurrently pushing objects on it. We also cannot
+ // check if the region stack is empty because if a thread is aborting
+ // it can push a partially done region back.
+ guarantee( _cm->out_of_regions() &&
+ _task_queue->size() == 0, "only way to reach here" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id);
+
+ _termination_start_time_ms = os::elapsedVTime() * 1000.0;
+ // The CMTask class also extends the TerminatorTerminator class,
+ // hence its should_exit_termination() method will also decide
+ // whether to exit the termination protocol or not.
+ bool finished = _cm->terminator()->offer_termination(this);
+ double termination_end_time_ms = os::elapsedVTime() * 1000.0;
+ _termination_time_ms +=
+ termination_end_time_ms - _termination_start_time_ms;
+
+ if (finished) {
+ // We're all done.
+
+ if (_task_id == 0) {
+ // let's allow task 0 to do this
+ if (concurrent()) {
+ guarantee( _cm->concurrent_marking_in_progress(), "invariant" );
+ // we need to set this to false before the next
+ // safepoint. This way we ensure that the marking phase
+ // doesn't observe any more heap expansions.
+ _cm->clear_concurrent_marking_in_progress();
+ }
+ }
+
+ // We can now guarantee that the global stack is empty, since
+ // all other tasks have finished.
+ guarantee( _cm->out_of_regions() &&
+ _cm->region_stack_empty() &&
+ _cm->mark_stack_empty() &&
+ _task_queue->size() == 0 &&
+ !_cm->has_overflown() &&
+ !_cm->mark_stack_overflow() &&
+ !_cm->region_stack_overflow(),
+ "only way to reach here" );
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id);
+ } else {
+ // Apparently there's more work to do. Let's abort this task. It
+ // will restart it and we can hopefully find more things to do.
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] apparently there is more work to do", _task_id);
+
+ set_has_aborted();
+ statsOnly( ++_aborted_termination );
+ }
+ }
+
+ // Mainly for debugging purposes to make sure that a pointer to the
+ // closure which was statically allocated in this frame doesn't
+ // escape it by accident.
+ set_oop_closure(NULL);
+ double end_time_ms = os::elapsedVTime() * 1000.0;
+ double elapsed_time_ms = end_time_ms - _start_time_ms;
+ // Update the step history.
+ _step_times_ms.add(elapsed_time_ms);
+
+ if (has_aborted()) {
+ // The task was aborted for some reason.
+
+ statsOnly( ++_aborted );
+
+ if (_has_aborted_timed_out) {
+ double diff_ms = elapsed_time_ms - _time_target_ms;
+ // Keep statistics of how well we did with respect to hitting
+ // our target only if we actually timed out (if we aborted for
+ // other reasons, then the results might get skewed).
+ _marking_step_diffs_ms.add(diff_ms);
+ }
+
+ if (_cm->has_overflown()) {
+ // This is the interesting one. We aborted because a global
+ // overflow was raised. This means we have to restart the
+ // marking phase and start iterating over regions. However, in
+ // order to do this we have to make sure that all tasks stop
+ // what they are doing and re-initialise in a safe manner. We
+ // will achieve this with the use of two barrier sync points.
+
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] detected overflow", _task_id);
+
+ _cm->enter_first_sync_barrier(_task_id);
+ // When we exit this sync barrier we know that all tasks have
+ // stopped doing marking work. So, it's now safe to
+ // re-initialise our data structures. At the end of this method,
+ // task 0 will clear the global data structures.
+
+ statsOnly( ++_aborted_overflow );
+
+ // We clear the local state of this task...
+ clear_region_fields();
+
+ // ...and enter the second barrier.
+ _cm->enter_second_sync_barrier(_task_id);
+ // At this point everything has bee re-initialised and we're
+ // ready to restart.
+ }
+
+ if (_cm->verbose_low()) {
+ gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, "
+ "elapsed = %1.2lfms <<<<<<<<<<",
+ _task_id, _time_target_ms, elapsed_time_ms);
+ if (_cm->has_aborted())
+ gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========",
+ _task_id);
+ }
+ } else {
+ if (_cm->verbose_low())
+ gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, "
+ "elapsed = %1.2lfms <<<<<<<<<<",
+ _task_id, _time_target_ms, elapsed_time_ms);
+ }
+
+ _claimed = false;
+}
+
+CMTask::CMTask(int task_id,
+ ConcurrentMark* cm,
+ CMTaskQueue* task_queue,
+ CMTaskQueueSet* task_queues)
+ : _g1h(G1CollectedHeap::heap()),
+ _co_tracker(G1CMGroup),
+ _task_id(task_id), _cm(cm),
+ _claimed(false),
+ _nextMarkBitMap(NULL), _hash_seed(17),
+ _task_queue(task_queue),
+ _task_queues(task_queues),
+ _oop_closure(NULL) {
+ guarantee( task_queue != NULL, "invariant" );
+ guarantee( task_queues != NULL, "invariant" );
+
+ statsOnly( _clock_due_to_scanning = 0;
+ _clock_due_to_marking = 0 );
+
+ _marking_step_diffs_ms.add(0.5);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp
new file mode 100644
index 00000000000..a572f74f3a7
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.hpp
@@ -0,0 +1,1049 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+class G1CollectedHeap;
+class CMTask;
+typedef GenericTaskQueue CMTaskQueue;
+typedef GenericTaskQueueSet CMTaskQueueSet;
+
+// A generic CM bit map. This is essentially a wrapper around the BitMap
+// class, with one bit per (1<<_shifter) HeapWords.
+
+class CMBitMapRO {
+ protected:
+ HeapWord* _bmStartWord; // base address of range covered by map
+ size_t _bmWordSize; // map size (in #HeapWords covered)
+ const int _shifter; // map to char or bit
+ VirtualSpace _virtual_space; // underlying the bit map
+ BitMap _bm; // the bit map itself
+
+ public:
+ // constructor
+ CMBitMapRO(ReservedSpace rs, int shifter);
+
+ enum { do_yield = true };
+
+ // inquiries
+ HeapWord* startWord() const { return _bmStartWord; }
+ size_t sizeInWords() const { return _bmWordSize; }
+ // the following is one past the last word in space
+ HeapWord* endWord() const { return _bmStartWord + _bmWordSize; }
+
+ // read marks
+
+ bool isMarked(HeapWord* addr) const {
+ assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+ "outside underlying space?");
+ return _bm.at(heapWordToOffset(addr));
+ }
+
+ // iteration
+ bool iterate(BitMapClosure* cl) { return _bm.iterate(cl); }
+ bool iterate(BitMapClosure* cl, MemRegion mr);
+
+ // Return the address corresponding to the next marked bit at or after
+ // "addr", and before "limit", if "limit" is non-NULL. If there is no
+ // such bit, returns "limit" if that is non-NULL, or else "endWord()".
+ HeapWord* getNextMarkedWordAddress(HeapWord* addr,
+ HeapWord* limit = NULL) const;
+ // Return the address corresponding to the next unmarked bit at or after
+ // "addr", and before "limit", if "limit" is non-NULL. If there is no
+ // such bit, returns "limit" if that is non-NULL, or else "endWord()".
+ HeapWord* getNextUnmarkedWordAddress(HeapWord* addr,
+ HeapWord* limit = NULL) const;
+
+ // conversion utilities
+ // XXX Fix these so that offsets are size_t's...
+ HeapWord* offsetToHeapWord(size_t offset) const {
+ return _bmStartWord + (offset << _shifter);
+ }
+ size_t heapWordToOffset(HeapWord* addr) const {
+ return pointer_delta(addr, _bmStartWord) >> _shifter;
+ }
+ int heapWordDiffToOffsetDiff(size_t diff) const;
+ HeapWord* nextWord(HeapWord* addr) {
+ return offsetToHeapWord(heapWordToOffset(addr) + 1);
+ }
+
+ void mostly_disjoint_range_union(BitMap* from_bitmap,
+ size_t from_start_index,
+ HeapWord* to_start_word,
+ size_t word_num);
+
+ // debugging
+ NOT_PRODUCT(bool covers(ReservedSpace rs) const;)
+};
+
+class CMBitMap : public CMBitMapRO {
+
+ public:
+ // constructor
+ CMBitMap(ReservedSpace rs, int shifter) :
+ CMBitMapRO(rs, shifter) {}
+
+ // write marks
+ void mark(HeapWord* addr) {
+ assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+ "outside underlying space?");
+ _bm.at_put(heapWordToOffset(addr), true);
+ }
+ void clear(HeapWord* addr) {
+ assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+ "outside underlying space?");
+ _bm.at_put(heapWordToOffset(addr), false);
+ }
+ bool parMark(HeapWord* addr) {
+ assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+ "outside underlying space?");
+ return _bm.par_at_put(heapWordToOffset(addr), true);
+ }
+ bool parClear(HeapWord* addr) {
+ assert(_bmStartWord <= addr && addr < (_bmStartWord + _bmWordSize),
+ "outside underlying space?");
+ return _bm.par_at_put(heapWordToOffset(addr), false);
+ }
+ void markRange(MemRegion mr);
+ void clearAll();
+ void clearRange(MemRegion mr);
+
+ // Starting at the bit corresponding to "addr" (inclusive), find the next
+ // "1" bit, if any. This bit starts some run of consecutive "1"'s; find
+ // the end of this run (stopping at "end_addr"). Return the MemRegion
+ // covering from the start of the region corresponding to the first bit
+ // of the run to the end of the region corresponding to the last bit of
+ // the run. If there is no "1" bit at or after "addr", return an empty
+ // MemRegion.
+ MemRegion getAndClearMarkedRegion(HeapWord* addr, HeapWord* end_addr);
+};
+
+// Represents a marking stack used by the CM collector.
+// Ideally this should be GrowableArray<> just like MSC's marking stack(s).
+class CMMarkStack {
+ ConcurrentMark* _cm;
+ oop* _base; // bottom of stack
+ jint _index; // one more than last occupied index
+ jint _capacity; // max #elements
+ jint _oops_do_bound; // Number of elements to include in next iteration.
+ NOT_PRODUCT(jint _max_depth;) // max depth plumbed during run
+
+ bool _overflow;
+ DEBUG_ONLY(bool _drain_in_progress;)
+ DEBUG_ONLY(bool _drain_in_progress_yields;)
+
+ public:
+ CMMarkStack(ConcurrentMark* cm);
+ ~CMMarkStack();
+
+ void allocate(size_t size);
+
+ oop pop() {
+ if (!isEmpty()) {
+ return _base[--_index] ;
+ }
+ return NULL;
+ }
+
+ // If overflow happens, don't do the push, and record the overflow.
+ // *Requires* that "ptr" is already marked.
+ void push(oop ptr) {
+ if (isFull()) {
+ // Record overflow.
+ _overflow = true;
+ return;
+ } else {
+ _base[_index++] = ptr;
+ NOT_PRODUCT(_max_depth = MAX2(_max_depth, _index));
+ }
+ }
+ // Non-block impl. Note: concurrency is allowed only with other
+ // "par_push" operations, not with "pop" or "drain". We would need
+ // parallel versions of them if such concurrency was desired.
+ void par_push(oop ptr);
+
+ // Pushes the first "n" elements of "ptr_arr" on the stack.
+ // Non-block impl. Note: concurrency is allowed only with other
+ // "par_adjoin_arr" or "push" operations, not with "pop" or "drain".
+ void par_adjoin_arr(oop* ptr_arr, int n);
+
+ // Pushes the first "n" elements of "ptr_arr" on the stack.
+ // Locking impl: concurrency is allowed only with
+ // "par_push_arr" and/or "par_pop_arr" operations, which use the same
+ // locking strategy.
+ void par_push_arr(oop* ptr_arr, int n);
+
+ // If returns false, the array was empty. Otherwise, removes up to "max"
+ // elements from the stack, and transfers them to "ptr_arr" in an
+ // unspecified order. The actual number transferred is given in "n" ("n
+ // == 0" is deliberately redundant with the return value.) Locking impl:
+ // concurrency is allowed only with "par_push_arr" and/or "par_pop_arr"
+ // operations, which use the same locking strategy.
+ bool par_pop_arr(oop* ptr_arr, int max, int* n);
+
+ // Drain the mark stack, applying the given closure to all fields of
+ // objects on the stack. (That is, continue until the stack is empty,
+ // even if closure applications add entries to the stack.) The "bm"
+ // argument, if non-null, may be used to verify that only marked objects
+ // are on the mark stack. If "yield_after" is "true", then the
+ // concurrent marker performing the drain offers to yield after
+ // processing each object. If a yield occurs, stops the drain operation
+ // and returns false. Otherwise, returns true.
+ template
+ bool drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after = false);
+
+ bool isEmpty() { return _index == 0; }
+ bool isFull() { return _index == _capacity; }
+ int maxElems() { return _capacity; }
+
+ bool overflow() { return _overflow; }
+ void clear_overflow() { _overflow = false; }
+
+ int size() { return _index; }
+
+ void setEmpty() { _index = 0; clear_overflow(); }
+
+ // Record the current size; a subsequent "oops_do" will iterate only over
+ // indices valid at the time of this call.
+ void set_oops_do_bound(jint bound = -1) {
+ if (bound == -1) {
+ _oops_do_bound = _index;
+ } else {
+ _oops_do_bound = bound;
+ }
+ }
+ jint oops_do_bound() { return _oops_do_bound; }
+ // iterate over the oops in the mark stack, up to the bound recorded via
+ // the call above.
+ void oops_do(OopClosure* f);
+};
+
+class CMRegionStack {
+ MemRegion* _base;
+ jint _capacity;
+ jint _index;
+ jint _oops_do_bound;
+ bool _overflow;
+public:
+ CMRegionStack();
+ ~CMRegionStack();
+ void allocate(size_t size);
+
+ // This is lock-free; assumes that it will only be called in parallel
+ // with other "push" operations (no pops).
+ void push(MemRegion mr);
+
+ // Lock-free; assumes that it will only be called in parallel
+ // with other "pop" operations (no pushes).
+ MemRegion pop();
+
+ bool isEmpty() { return _index == 0; }
+ bool isFull() { return _index == _capacity; }
+
+ bool overflow() { return _overflow; }
+ void clear_overflow() { _overflow = false; }
+
+ int size() { return _index; }
+
+ // It iterates over the entries in the region stack and it
+ // invalidates (i.e. assigns MemRegion()) the ones that point to
+ // regions in the collection set.
+ bool invalidate_entries_into_cset();
+
+ // This gives an upper bound up to which the iteration in
+ // invalidate_entries_into_cset() will reach. This prevents
+ // newly-added entries to be unnecessarily scanned.
+ void set_oops_do_bound() {
+ _oops_do_bound = _index;
+ }
+
+ void setEmpty() { _index = 0; clear_overflow(); }
+};
+
+// this will enable a variety of different statistics per GC task
+#define _MARKING_STATS_ 0
+// this will enable the higher verbose levels
+#define _MARKING_VERBOSE_ 0
+
+#if _MARKING_STATS_
+#define statsOnly(statement) \
+do { \
+ statement ; \
+} while (0)
+#else // _MARKING_STATS_
+#define statsOnly(statement) \
+do { \
+} while (0)
+#endif // _MARKING_STATS_
+
+// Some extra guarantees that I like to also enable in optimised mode
+// when debugging. If you want to enable them, comment out the assert
+// macro and uncomment out the guaratee macro
+// #define tmp_guarantee_CM(expr, str) guarantee(expr, str)
+#define tmp_guarantee_CM(expr, str) assert(expr, str)
+
+typedef enum {
+ no_verbose = 0, // verbose turned off
+ stats_verbose, // only prints stats at the end of marking
+ low_verbose, // low verbose, mostly per region and per major event
+ medium_verbose, // a bit more detailed than low
+ high_verbose // per object verbose
+} CMVerboseLevel;
+
+
+class ConcurrentMarkThread;
+
+class ConcurrentMark {
+ friend class ConcurrentMarkThread;
+ friend class CMTask;
+ friend class CMBitMapClosure;
+ friend class CSMarkOopClosure;
+ friend class CMGlobalObjectClosure;
+ friend class CMRemarkTask;
+ friend class CMConcurrentMarkingTask;
+ friend class G1ParNoteEndTask;
+ friend class CalcLiveObjectsClosure;
+
+protected:
+ ConcurrentMarkThread* _cmThread; // the thread doing the work
+ G1CollectedHeap* _g1h; // the heap.
+ size_t _parallel_marking_threads; // the number of marking
+ // threads we'll use
+ double _sleep_factor; // how much we have to sleep, with
+ // respect to the work we just did, to
+ // meet the marking overhead goal
+ double _marking_task_overhead; // marking target overhead for
+ // a single task
+
+ // same as the two above, but for the cleanup task
+ double _cleanup_sleep_factor;
+ double _cleanup_task_overhead;
+
+ // Stuff related to age cohort processing.
+ struct ParCleanupThreadState {
+ char _pre[64];
+ UncleanRegionList list;
+ char _post[64];
+ };
+ ParCleanupThreadState** _par_cleanup_thread_state;
+
+ // CMS marking support structures
+ CMBitMap _markBitMap1;
+ CMBitMap _markBitMap2;
+ CMBitMapRO* _prevMarkBitMap; // completed mark bitmap
+ CMBitMap* _nextMarkBitMap; // under-construction mark bitmap
+ bool _at_least_one_mark_complete;
+
+ BitMap _region_bm;
+ BitMap _card_bm;
+
+ // Heap bounds
+ HeapWord* _heap_start;
+ HeapWord* _heap_end;
+
+ // For gray objects
+ CMMarkStack _markStack; // Grey objects behind global finger.
+ CMRegionStack _regionStack; // Grey regions behind global finger.
+ HeapWord* volatile _finger; // the global finger, region aligned,
+ // always points to the end of the
+ // last claimed region
+
+ // marking tasks
+ size_t _max_task_num; // maximum task number
+ size_t _active_tasks; // task num currently active
+ CMTask** _tasks; // task queue array (max_task_num len)
+ CMTaskQueueSet* _task_queues; // task queue set
+ ParallelTaskTerminator _terminator; // for termination
+
+ // Two sync barriers that are used to synchronise tasks when an
+ // overflow occurs. The algorithm is the following. All tasks enter
+ // the first one to ensure that they have all stopped manipulating
+ // the global data structures. After they exit it, they re-initialise
+ // their data structures and task 0 re-initialises the global data
+ // structures. Then, they enter the second sync barrier. This
+ // ensure, that no task starts doing work before all data
+ // structures (local and global) have been re-initialised. When they
+ // exit it, they are free to start working again.
+ WorkGangBarrierSync _first_overflow_barrier_sync;
+ WorkGangBarrierSync _second_overflow_barrier_sync;
+
+
+ // this is set by any task, when an overflow on the global data
+ // structures is detected.
+ volatile bool _has_overflown;
+ // true: marking is concurrent, false: we're in remark
+ volatile bool _concurrent;
+ // set at the end of a Full GC so that marking aborts
+ volatile bool _has_aborted;
+ // used when remark aborts due to an overflow to indicate that
+ // another concurrent marking phase should start
+ volatile bool _restart_for_overflow;
+
+ // This is true from the very start of concurrent marking until the
+ // point when all the tasks complete their work. It is really used
+ // to determine the points between the end of concurrent marking and
+ // time of remark.
+ volatile bool _concurrent_marking_in_progress;
+
+ // verbose level
+ CMVerboseLevel _verbose_level;
+
+ COTracker _cleanup_co_tracker;
+
+ // These two fields are used to implement the optimisation that
+ // avoids pushing objects on the global/region stack if there are
+ // no collection set regions above the lowest finger.
+
+ // This is the lowest finger (among the global and local fingers),
+ // which is calculated before a new collection set is chosen.
+ HeapWord* _min_finger;
+ // If this flag is true, objects/regions that are marked below the
+ // finger should be pushed on the stack(s). If this is flag is
+ // false, it is safe not to push them on the stack(s).
+ bool _should_gray_objects;
+
+ // All of these times are in ms.
+ NumberSeq _init_times;
+ NumberSeq _remark_times;
+ NumberSeq _remark_mark_times;
+ NumberSeq _remark_weak_ref_times;
+ NumberSeq _cleanup_times;
+ double _total_counting_time;
+ double _total_rs_scrub_time;
+
+ double* _accum_task_vtime; // accumulated task vtime
+
+ WorkGang* _parallel_workers;
+
+ void weakRefsWork(bool clear_all_soft_refs);
+
+ void swapMarkBitMaps();
+
+ // It resets the global marking data structures, as well as the
+ // task local ones; should be called during initial mark.
+ void reset();
+ // It resets all the marking data structures.
+ void clear_marking_state();
+
+ // It should be called to indicate which phase we're in (concurrent
+ // mark or remark) and how many threads are currently active.
+ void set_phase(size_t active_tasks, bool concurrent);
+ // We do this after we're done with marking so that the marking data
+ // structures are initialised to a sensible and predictable state.
+ void set_non_marking_state();
+
+ // prints all gathered CM-related statistics
+ void print_stats();
+
+ // accessor methods
+ size_t parallel_marking_threads() { return _parallel_marking_threads; }
+ double sleep_factor() { return _sleep_factor; }
+ double marking_task_overhead() { return _marking_task_overhead;}
+ double cleanup_sleep_factor() { return _cleanup_sleep_factor; }
+ double cleanup_task_overhead() { return _cleanup_task_overhead;}
+
+ HeapWord* finger() { return _finger; }
+ bool concurrent() { return _concurrent; }
+ size_t active_tasks() { return _active_tasks; }
+ ParallelTaskTerminator* terminator() { return &_terminator; }
+
+ // It claims the next available region to be scanned by a marking
+ // task. It might return NULL if the next region is empty or we have
+ // run out of regions. In the latter case, out_of_regions()
+ // determines whether we've really run out of regions or the task
+ // should call claim_region() again. This might seem a bit
+ // awkward. Originally, the code was written so that claim_region()
+ // either successfully returned with a non-empty region or there
+ // were no more regions to be claimed. The problem with this was
+ // that, in certain circumstances, it iterated over large chunks of
+ // the heap finding only empty regions and, while it was working, it
+ // was preventing the calling task to call its regular clock
+ // method. So, this way, each task will spend very little time in
+ // claim_region() and is allowed to call the regular clock method
+ // frequently.
+ HeapRegion* claim_region(int task);
+
+ // It determines whether we've run out of regions to scan.
+ bool out_of_regions() { return _finger == _heap_end; }
+
+ // Returns the task with the given id
+ CMTask* task(int id) {
+ guarantee( 0 <= id && id < (int) _active_tasks, "task id not within "
+ "active bounds" );
+ return _tasks[id];
+ }
+
+ // Returns the task queue with the given id
+ CMTaskQueue* task_queue(int id) {
+ guarantee( 0 <= id && id < (int) _active_tasks, "task queue id not within "
+ "active bounds" );
+ return (CMTaskQueue*) _task_queues->queue(id);
+ }
+
+ // Returns the task queue set
+ CMTaskQueueSet* task_queues() { return _task_queues; }
+
+ // Access / manipulation of the overflow flag which is set to
+ // indicate that the global stack or region stack has overflown
+ bool has_overflown() { return _has_overflown; }
+ void set_has_overflown() { _has_overflown = true; }
+ void clear_has_overflown() { _has_overflown = false; }
+
+ bool has_aborted() { return _has_aborted; }
+ bool restart_for_overflow() { return _restart_for_overflow; }
+
+ // Methods to enter the two overflow sync barriers
+ void enter_first_sync_barrier(int task_num);
+ void enter_second_sync_barrier(int task_num);
+
+public:
+ // Manipulation of the global mark stack.
+ // Notice that the first mark_stack_push is CAS-based, whereas the
+ // two below are Mutex-based. This is OK since the first one is only
+ // called during evacuation pauses and doesn't compete with the
+ // other two (which are called by the marking tasks during
+ // concurrent marking or remark).
+ bool mark_stack_push(oop p) {
+ _markStack.par_push(p);
+ if (_markStack.overflow()) {
+ set_has_overflown();
+ return false;
+ }
+ return true;
+ }
+ bool mark_stack_push(oop* arr, int n) {
+ _markStack.par_push_arr(arr, n);
+ if (_markStack.overflow()) {
+ set_has_overflown();
+ return false;
+ }
+ return true;
+ }
+ void mark_stack_pop(oop* arr, int max, int* n) {
+ _markStack.par_pop_arr(arr, max, n);
+ }
+ size_t mark_stack_size() { return _markStack.size(); }
+ size_t partial_mark_stack_size_target() { return _markStack.maxElems()/3; }
+ bool mark_stack_overflow() { return _markStack.overflow(); }
+ bool mark_stack_empty() { return _markStack.isEmpty(); }
+
+ // Manipulation of the region stack
+ bool region_stack_push(MemRegion mr) {
+ _regionStack.push(mr);
+ if (_regionStack.overflow()) {
+ set_has_overflown();
+ return false;
+ }
+ return true;
+ }
+ MemRegion region_stack_pop() { return _regionStack.pop(); }
+ int region_stack_size() { return _regionStack.size(); }
+ bool region_stack_overflow() { return _regionStack.overflow(); }
+ bool region_stack_empty() { return _regionStack.isEmpty(); }
+
+ bool concurrent_marking_in_progress() {
+ return _concurrent_marking_in_progress;
+ }
+ void set_concurrent_marking_in_progress() {
+ _concurrent_marking_in_progress = true;
+ }
+ void clear_concurrent_marking_in_progress() {
+ _concurrent_marking_in_progress = false;
+ }
+
+ void update_accum_task_vtime(int i, double vtime) {
+ _accum_task_vtime[i] += vtime;
+ }
+
+ double all_task_accum_vtime() {
+ double ret = 0.0;
+ for (int i = 0; i < (int)_max_task_num; ++i)
+ ret += _accum_task_vtime[i];
+ return ret;
+ }
+
+ // Attempts to steal an object from the task queues of other tasks
+ bool try_stealing(int task_num, int* hash_seed, oop& obj) {
+ return _task_queues->steal(task_num, hash_seed, obj);
+ }
+
+ // It grays an object by first marking it. Then, if it's behind the
+ // global finger, it also pushes it on the global stack.
+ void deal_with_reference(oop obj);
+
+ ConcurrentMark(ReservedSpace rs, int max_regions);
+ ~ConcurrentMark();
+ ConcurrentMarkThread* cmThread() { return _cmThread; }
+
+ CMBitMapRO* prevMarkBitMap() const { return _prevMarkBitMap; }
+ CMBitMap* nextMarkBitMap() const { return _nextMarkBitMap; }
+
+ // The following three are interaction between CM and
+ // G1CollectedHeap
+
+ // This notifies CM that a root during initial-mark needs to be
+ // grayed and it's MT-safe. Currently, we just mark it. But, in the
+ // future, we can experiment with pushing it on the stack and we can
+ // do this without changing G1CollectedHeap.
+ void grayRoot(oop p);
+ // It's used during evacuation pauses to gray a region, if
+ // necessary, and it's MT-safe. It assumes that the caller has
+ // marked any objects on that region. If _should_gray_objects is
+ // true and we're still doing concurrent marking, the region is
+ // pushed on the region stack, if it is located below the global
+ // finger, otherwise we do nothing.
+ void grayRegionIfNecessary(MemRegion mr);
+ // It's used during evacuation pauses to mark and, if necessary,
+ // gray a single object and it's MT-safe. It assumes the caller did
+ // not mark the object. If _should_gray_objects is true and we're
+ // still doing concurrent marking, the objects is pushed on the
+ // global stack, if it is located below the global finger, otherwise
+ // we do nothing.
+ void markAndGrayObjectIfNecessary(oop p);
+
+ // This iterates over the bitmap of the previous marking and prints
+ // out all objects that are marked on the bitmap and indicates
+ // whether what they point to is also marked or not.
+ void print_prev_bitmap_reachable();
+
+ // Clear the next marking bitmap (will be called concurrently).
+ void clearNextBitmap();
+
+ // main CMS steps and related support
+ void checkpointRootsInitial();
+
+ // These two do the work that needs to be done before and after the
+ // initial root checkpoint. Since this checkpoint can be done at two
+ // different points (i.e. an explicit pause or piggy-backed on a
+ // young collection), then it's nice to be able to easily share the
+ // pre/post code. It might be the case that we can put everything in
+ // the post method. TP
+ void checkpointRootsInitialPre();
+ void checkpointRootsInitialPost();
+
+ // Do concurrent phase of marking, to a tentative transitive closure.
+ void markFromRoots();
+
+ // Process all unprocessed SATB buffers. It is called at the
+ // beginning of an evacuation pause.
+ void drainAllSATBBuffers();
+
+ void checkpointRootsFinal(bool clear_all_soft_refs);
+ void checkpointRootsFinalWork();
+ void calcDesiredRegions();
+ void cleanup();
+ void completeCleanup();
+
+ // Mark in the previous bitmap. NB: this is usually read-only, so use
+ // this carefully!
+ void markPrev(oop p);
+ void clear(oop p);
+ // Clears marks for all objects in the given range, for both prev and
+ // next bitmaps. NB: the previous bitmap is usually read-only, so use
+ // this carefully!
+ void clearRangeBothMaps(MemRegion mr);
+
+ // Record the current top of the mark and region stacks; a
+ // subsequent oops_do() on the mark stack and
+ // invalidate_entries_into_cset() on the region stack will iterate
+ // only over indices valid at the time of this call.
+ void set_oops_do_bound() {
+ _markStack.set_oops_do_bound();
+ _regionStack.set_oops_do_bound();
+ }
+ // Iterate over the oops in the mark stack and all local queues. It
+ // also calls invalidate_entries_into_cset() on the region stack.
+ void oops_do(OopClosure* f);
+ // It is called at the end of an evacuation pause during marking so
+ // that CM is notified of where the new end of the heap is. It
+ // doesn't do anything if concurrent_marking_in_progress() is false,
+ // unless the force parameter is true.
+ void update_g1_committed(bool force = false);
+
+ void complete_marking_in_collection_set();
+
+ // It indicates that a new collection set is being chosen.
+ void newCSet();
+ // It registers a collection set heap region with CM. This is used
+ // to determine whether any heap regions are located above the finger.
+ void registerCSetRegion(HeapRegion* hr);
+
+ // Returns "true" if at least one mark has been completed.
+ bool at_least_one_mark_complete() { return _at_least_one_mark_complete; }
+
+ bool isMarked(oop p) const {
+ assert(p != NULL && p->is_oop(), "expected an oop");
+ HeapWord* addr = (HeapWord*)p;
+ assert(addr >= _nextMarkBitMap->startWord() ||
+ addr < _nextMarkBitMap->endWord(), "in a region");
+
+ return _nextMarkBitMap->isMarked(addr);
+ }
+
+ inline bool not_yet_marked(oop p) const;
+
+ // XXX Debug code
+ bool containing_card_is_marked(void* p);
+ bool containing_cards_are_marked(void* start, void* last);
+
+ bool isPrevMarked(oop p) const {
+ assert(p != NULL && p->is_oop(), "expected an oop");
+ HeapWord* addr = (HeapWord*)p;
+ assert(addr >= _prevMarkBitMap->startWord() ||
+ addr < _prevMarkBitMap->endWord(), "in a region");
+
+ return _prevMarkBitMap->isMarked(addr);
+ }
+
+ inline bool do_yield_check(int worker_i = 0);
+ inline bool should_yield();
+
+ // Called to abort the marking cycle after a Full GC takes palce.
+ void abort();
+
+ void disable_co_trackers();
+
+ // This prints the global/local fingers. It is used for debugging.
+ NOT_PRODUCT(void print_finger();)
+
+ void print_summary_info();
+
+ // The following indicate whether a given verbose level has been
+ // set. Notice that anything above stats is conditional to
+ // _MARKING_VERBOSE_ having been set to 1
+ bool verbose_stats()
+ { return _verbose_level >= stats_verbose; }
+ bool verbose_low()
+ { return _MARKING_VERBOSE_ && _verbose_level >= low_verbose; }
+ bool verbose_medium()
+ { return _MARKING_VERBOSE_ && _verbose_level >= medium_verbose; }
+ bool verbose_high()
+ { return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; }
+};
+
+// A class representing a marking task.
+class CMTask : public TerminatorTerminator {
+private:
+ enum PrivateConstants {
+ // the regular clock call is called once the scanned words reaches
+ // this limit
+ words_scanned_period = 12*1024,
+ // the regular clock call is called once the number of visited
+ // references reaches this limit
+ refs_reached_period = 384,
+ // initial value for the hash seed, used in the work stealing code
+ init_hash_seed = 17,
+ // how many entries will be transferred between global stack and
+ // local queues
+ global_stack_transfer_size = 16
+ };
+
+ int _task_id;
+ G1CollectedHeap* _g1h;
+ ConcurrentMark* _cm;
+ CMBitMap* _nextMarkBitMap;
+ // the task queue of this task
+ CMTaskQueue* _task_queue;
+ // the task queue set---needed for stealing
+ CMTaskQueueSet* _task_queues;
+ // indicates whether the task has been claimed---this is only for
+ // debugging purposes
+ bool _claimed;
+
+ // number of calls to this task
+ int _calls;
+
+ // concurrent overhead over a single CPU for this task
+ COTracker _co_tracker;
+
+ // when the virtual timer reaches this time, the marking step should
+ // exit
+ double _time_target_ms;
+ // the start time of the current marking step
+ double _start_time_ms;
+
+ // the oop closure used for iterations over oops
+ OopClosure* _oop_closure;
+
+ // the region this task is scanning, NULL if we're not scanning any
+ HeapRegion* _curr_region;
+ // the local finger of this task, NULL if we're not scanning a region
+ HeapWord* _finger;
+ // limit of the region this task is scanning, NULL if we're not scanning one
+ HeapWord* _region_limit;
+
+ // This is used only when we scan regions popped from the region
+ // stack. It records what the last object on such a region we
+ // scanned was. It is used to ensure that, if we abort region
+ // iteration, we do not rescan the first part of the region. This
+ // should be NULL when we're not scanning a region from the region
+ // stack.
+ HeapWord* _region_finger;
+
+ // the number of words this task has scanned
+ size_t _words_scanned;
+ // When _words_scanned reaches this limit, the regular clock is
+ // called. Notice that this might be decreased under certain
+ // circumstances (i.e. when we believe that we did an expensive
+ // operation).
+ size_t _words_scanned_limit;
+ // the initial value of _words_scanned_limit (i.e. what it was
+ // before it was decreased).
+ size_t _real_words_scanned_limit;
+
+ // the number of references this task has visited
+ size_t _refs_reached;
+ // When _refs_reached reaches this limit, the regular clock is
+ // called. Notice this this might be decreased under certain
+ // circumstances (i.e. when we believe that we did an expensive
+ // operation).
+ size_t _refs_reached_limit;
+ // the initial value of _refs_reached_limit (i.e. what it was before
+ // it was decreased).
+ size_t _real_refs_reached_limit;
+
+ // used by the work stealing stuff
+ int _hash_seed;
+ // if this is true, then the task has aborted for some reason
+ bool _has_aborted;
+ // set when the task aborts because it has met its time quota
+ bool _has_aborted_timed_out;
+ // true when we're draining SATB buffers; this avoids the task
+ // aborting due to SATB buffers being available (as we're already
+ // dealing with them)
+ bool _draining_satb_buffers;
+
+ // number sequence of past step times
+ NumberSeq _step_times_ms;
+ // elapsed time of this task
+ double _elapsed_time_ms;
+ // termination time of this task
+ double _termination_time_ms;
+ // when this task got into the termination protocol
+ double _termination_start_time_ms;
+
+ // true when the task is during a concurrent phase, false when it is
+ // in the remark phase (so, in the latter case, we do not have to
+ // check all the things that we have to check during the concurrent
+ // phase, i.e. SATB buffer availability...)
+ bool _concurrent;
+
+ TruncatedSeq _marking_step_diffs_ms;
+
+ // LOTS of statistics related with this task
+#if _MARKING_STATS_
+ NumberSeq _all_clock_intervals_ms;
+ double _interval_start_time_ms;
+
+ int _aborted;
+ int _aborted_overflow;
+ int _aborted_cm_aborted;
+ int _aborted_yield;
+ int _aborted_timed_out;
+ int _aborted_satb;
+ int _aborted_termination;
+
+ int _steal_attempts;
+ int _steals;
+
+ int _clock_due_to_marking;
+ int _clock_due_to_scanning;
+
+ int _local_pushes;
+ int _local_pops;
+ int _local_max_size;
+ int _objs_scanned;
+
+ int _global_pushes;
+ int _global_pops;
+ int _global_max_size;
+
+ int _global_transfers_to;
+ int _global_transfers_from;
+
+ int _region_stack_pops;
+
+ int _regions_claimed;
+ int _objs_found_on_bitmap;
+
+ int _satb_buffers_processed;
+#endif // _MARKING_STATS_
+
+ // it updates the local fields after this task has claimed
+ // a new region to scan
+ void setup_for_region(HeapRegion* hr);
+ // it brings up-to-date the limit of the region
+ void update_region_limit();
+ // it resets the local fields after a task has finished scanning a
+ // region
+ void giveup_current_region();
+
+ // called when either the words scanned or the refs visited limit
+ // has been reached
+ void reached_limit();
+ // recalculates the words scanned and refs visited limits
+ void recalculate_limits();
+ // decreases the words scanned and refs visited limits when we reach
+ // an expensive operation
+ void decrease_limits();
+ // it checks whether the words scanned or refs visited reached their
+ // respective limit and calls reached_limit() if they have
+ void check_limits() {
+ if (_words_scanned >= _words_scanned_limit ||
+ _refs_reached >= _refs_reached_limit)
+ reached_limit();
+ }
+ // this is supposed to be called regularly during a marking step as
+ // it checks a bunch of conditions that might cause the marking step
+ // to abort
+ void regular_clock_call();
+ bool concurrent() { return _concurrent; }
+
+public:
+ // It resets the task; it should be called right at the beginning of
+ // a marking phase.
+ void reset(CMBitMap* _nextMarkBitMap);
+ // it clears all the fields that correspond to a claimed region.
+ void clear_region_fields();
+
+ void set_concurrent(bool concurrent) { _concurrent = concurrent; }
+
+ void enable_co_tracker() {
+ guarantee( !_co_tracker.enabled(), "invariant" );
+ _co_tracker.enable();
+ }
+ void disable_co_tracker() {
+ guarantee( _co_tracker.enabled(), "invariant" );
+ _co_tracker.disable();
+ }
+ bool co_tracker_enabled() {
+ return _co_tracker.enabled();
+ }
+ void reset_co_tracker(double starting_conc_overhead = 0.0) {
+ _co_tracker.reset(starting_conc_overhead);
+ }
+ void start_co_tracker() {
+ _co_tracker.start();
+ }
+ void update_co_tracker(bool force_end = false) {
+ _co_tracker.update(force_end);
+ }
+
+ // The main method of this class which performs a marking step
+ // trying not to exceed the given duration. However, it might exit
+ // prematurely, according to some conditions (i.e. SATB buffers are
+ // available for processing).
+ void do_marking_step(double target_ms);
+
+ // These two calls start and stop the timer
+ void record_start_time() {
+ _elapsed_time_ms = os::elapsedTime() * 1000.0;
+ }
+ void record_end_time() {
+ _elapsed_time_ms = os::elapsedTime() * 1000.0 - _elapsed_time_ms;
+ }
+
+ // returns the task ID
+ int task_id() { return _task_id; }
+
+ // From TerminatorTerminator. It determines whether this task should
+ // exit the termination protocol after it's entered it.
+ virtual bool should_exit_termination();
+
+ HeapWord* finger() { return _finger; }
+
+ bool has_aborted() { return _has_aborted; }
+ void set_has_aborted() { _has_aborted = true; }
+ void clear_has_aborted() { _has_aborted = false; }
+ bool claimed() { return _claimed; }
+
+ void set_oop_closure(OopClosure* oop_closure) {
+ _oop_closure = oop_closure;
+ }
+
+ // It grays the object by marking it and, if necessary, pushing it
+ // on the local queue
+ void deal_with_reference(oop obj);
+
+ // It scans an object and visits its children.
+ void scan_object(oop obj) {
+ tmp_guarantee_CM( _nextMarkBitMap->isMarked((HeapWord*) obj),
+ "invariant" );
+
+ if (_cm->verbose_high())
+ gclog_or_tty->print_cr("[%d] we're scanning object "PTR_FORMAT,
+ _task_id, (void*) obj);
+
+ size_t obj_size = obj->size();
+ _words_scanned += obj_size;
+
+ obj->oop_iterate(_oop_closure);
+ statsOnly( ++_objs_scanned );
+ check_limits();
+ }
+
+ // It pushes an object on the local queue.
+ void push(oop obj);
+
+ // These two move entries to/from the global stack.
+ void move_entries_to_global_stack();
+ void get_entries_from_global_stack();
+
+ // It pops and scans objects from the local queue. If partially is
+ // true, then it stops when the queue size is of a given limit. If
+ // partially is false, then it stops when the queue is empty.
+ void drain_local_queue(bool partially);
+ // It moves entries from the global stack to the local queue and
+ // drains the local queue. If partially is true, then it stops when
+ // both the global stack and the local queue reach a given size. If
+ // partially if false, it tries to empty them totally.
+ void drain_global_stack(bool partially);
+ // It keeps picking SATB buffers and processing them until no SATB
+ // buffers are available.
+ void drain_satb_buffers();
+ // It keeps popping regions from the region stack and processing
+ // them until the region stack is empty.
+ void drain_region_stack(BitMapClosure* closure);
+
+ // moves the local finger to a new location
+ inline void move_finger_to(HeapWord* new_finger) {
+ tmp_guarantee_CM( new_finger >= _finger && new_finger < _region_limit,
+ "invariant" );
+ _finger = new_finger;
+ }
+
+ // moves the region finger to a new location
+ inline void move_region_finger_to(HeapWord* new_finger) {
+ tmp_guarantee_CM( new_finger < _cm->finger(), "invariant" );
+ _region_finger = new_finger;
+ }
+
+ CMTask(int task_num, ConcurrentMark *cm,
+ CMTaskQueue* task_queue, CMTaskQueueSet* task_queues);
+
+ // it prints statistics associated with this task
+ void print_stats();
+
+#if _MARKING_STATS_
+ void increase_objs_found_on_bitmap() { ++_objs_found_on_bitmap; }
+#endif // _MARKING_STATS_
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp
new file mode 100644
index 00000000000..e26df0caae0
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp
@@ -0,0 +1,336 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_concurrentMarkThread.cpp.incl"
+
+// ======= Concurrent Mark Thread ========
+
+// The CM thread is created when the G1 garbage collector is used
+
+SurrogateLockerThread*
+ ConcurrentMarkThread::_slt = NULL;
+
+ConcurrentMarkThread::ConcurrentMarkThread(ConcurrentMark* cm) :
+ ConcurrentGCThread(),
+ _cm(cm),
+ _started(false),
+ _in_progress(false),
+ _vtime_accum(0.0),
+ _vtime_mark_accum(0.0),
+ _vtime_count_accum(0.0)
+{
+ create_and_start();
+}
+
+class CMCheckpointRootsInitialClosure: public VoidClosure {
+
+ ConcurrentMark* _cm;
+public:
+
+ CMCheckpointRootsInitialClosure(ConcurrentMark* cm) :
+ _cm(cm) {}
+
+ void do_void(){
+ _cm->checkpointRootsInitial();
+ }
+};
+
+class CMCheckpointRootsFinalClosure: public VoidClosure {
+
+ ConcurrentMark* _cm;
+public:
+
+ CMCheckpointRootsFinalClosure(ConcurrentMark* cm) :
+ _cm(cm) {}
+
+ void do_void(){
+ _cm->checkpointRootsFinal(false); // !clear_all_soft_refs
+ }
+};
+
+class CMCleanUp: public VoidClosure {
+ ConcurrentMark* _cm;
+public:
+
+ CMCleanUp(ConcurrentMark* cm) :
+ _cm(cm) {}
+
+ void do_void(){
+ _cm->cleanup();
+ }
+};
+
+
+
+void ConcurrentMarkThread::run() {
+ initialize_in_thread();
+ _vtime_start = os::elapsedVTime();
+ wait_for_universe_init();
+
+ G1CollectedHeap* g1 = G1CollectedHeap::heap();
+ G1CollectorPolicy* g1_policy = g1->g1_policy();
+ G1MMUTracker *mmu_tracker = g1_policy->mmu_tracker();
+ Thread *current_thread = Thread::current();
+
+ while (!_should_terminate) {
+ // wait until started is set.
+ sleepBeforeNextCycle();
+ {
+ ResourceMark rm;
+ HandleMark hm;
+ double cycle_start = os::elapsedVTime();
+ double mark_start_sec = os::elapsedTime();
+ char verbose_str[128];
+
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ tty->print_cr("[GC concurrent-mark-start]");
+ }
+
+ if (!g1_policy->in_young_gc_mode()) {
+ // this ensures the flag is not set if we bail out of the marking
+ // cycle; normally the flag is cleared immediately after cleanup
+ g1->set_marking_complete();
+
+ if (g1_policy->adaptive_young_list_length()) {
+ double now = os::elapsedTime();
+ double init_prediction_ms = g1_policy->predict_init_time_ms();
+ jlong sleep_time_ms = mmu_tracker->when_ms(now, init_prediction_ms);
+ os::sleep(current_thread, sleep_time_ms, false);
+ }
+
+ // We don't have to skip here if we've been asked to restart, because
+ // in the worst case we just enqueue a new VM operation to start a
+ // marking. Note that the init operation resets has_aborted()
+ CMCheckpointRootsInitialClosure init_cl(_cm);
+ strcpy(verbose_str, "GC initial-mark");
+ VM_CGC_Operation op(&init_cl, verbose_str);
+ VMThread::execute(&op);
+ }
+
+ int iter = 0;
+ do {
+ iter++;
+ if (!cm()->has_aborted()) {
+ _cm->markFromRoots();
+ } else {
+ if (TraceConcurrentMark)
+ gclog_or_tty->print_cr("CM-skip-mark-from-roots");
+ }
+
+ double mark_end_time = os::elapsedVTime();
+ double mark_end_sec = os::elapsedTime();
+ _vtime_mark_accum += (mark_end_time - cycle_start);
+ if (!cm()->has_aborted()) {
+ if (g1_policy->adaptive_young_list_length()) {
+ double now = os::elapsedTime();
+ double remark_prediction_ms = g1_policy->predict_remark_time_ms();
+ jlong sleep_time_ms = mmu_tracker->when_ms(now, remark_prediction_ms);
+ os::sleep(current_thread, sleep_time_ms, false);
+ }
+
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-mark-end, %1.7lf sec]",
+ mark_end_sec - mark_start_sec);
+ }
+
+ CMCheckpointRootsFinalClosure final_cl(_cm);
+ sprintf(verbose_str, "GC remark");
+ VM_CGC_Operation op(&final_cl, verbose_str);
+ VMThread::execute(&op);
+ } else {
+ if (TraceConcurrentMark)
+ gclog_or_tty->print_cr("CM-skip-remark");
+ }
+ if (cm()->restart_for_overflow() &&
+ G1TraceMarkStackOverflow) {
+ gclog_or_tty->print_cr("Restarting conc marking because of MS overflow "
+ "in remark (restart #%d).", iter);
+ }
+
+ if (cm()->restart_for_overflow()) {
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-mark-restart-for-overflow]");
+ }
+ }
+ } while (cm()->restart_for_overflow());
+ double counting_start_time = os::elapsedVTime();
+
+ // YSR: These look dubious (i.e. redundant) !!! FIX ME
+ slt()->manipulatePLL(SurrogateLockerThread::acquirePLL);
+ slt()->manipulatePLL(SurrogateLockerThread::releaseAndNotifyPLL);
+
+ if (!cm()->has_aborted()) {
+ double count_start_sec = os::elapsedTime();
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-count-start]");
+ }
+
+ _sts.join();
+ _cm->calcDesiredRegions();
+ _sts.leave();
+
+ if (!cm()->has_aborted()) {
+ double count_end_sec = os::elapsedTime();
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-count-end, %1.7lf]",
+ count_end_sec - count_start_sec);
+ }
+ }
+ } else {
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-skip-end-game");
+ }
+ double end_time = os::elapsedVTime();
+ _vtime_count_accum += (end_time - counting_start_time);
+ // Update the total virtual time before doing this, since it will try
+ // to measure it to get the vtime for this marking. We purposely
+ // neglect the presumably-short "completeCleanup" phase here.
+ _vtime_accum = (end_time - _vtime_start);
+ if (!cm()->has_aborted()) {
+ if (g1_policy->adaptive_young_list_length()) {
+ double now = os::elapsedTime();
+ double cleanup_prediction_ms = g1_policy->predict_cleanup_time_ms();
+ jlong sleep_time_ms = mmu_tracker->when_ms(now, cleanup_prediction_ms);
+ os::sleep(current_thread, sleep_time_ms, false);
+ }
+
+ CMCleanUp cl_cl(_cm);
+ sprintf(verbose_str, "GC cleanup");
+ VM_CGC_Operation op(&cl_cl, verbose_str);
+ VMThread::execute(&op);
+ } else {
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-skip-cleanup");
+ G1CollectedHeap::heap()->set_marking_complete();
+ }
+
+ if (!cm()->has_aborted()) {
+ double cleanup_start_sec = os::elapsedTime();
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-cleanup-start]");
+ }
+
+ // Now do the remainder of the cleanup operation.
+ _sts.join();
+ _cm->completeCleanup();
+ if (!cm()->has_aborted()) {
+ g1_policy->record_concurrent_mark_cleanup_completed();
+
+ double cleanup_end_sec = os::elapsedTime();
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-cleanup-end, %1.7lf]",
+ cleanup_end_sec - cleanup_start_sec);
+ }
+ }
+ _sts.leave();
+ }
+ // We're done: no more unclean regions coming.
+ G1CollectedHeap::heap()->set_unclean_regions_coming(false);
+
+ if (cm()->has_aborted()) {
+ if (PrintGC) {
+ gclog_or_tty->date_stamp(PrintGCDateStamps);
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print_cr("[GC concurrent-mark-abort]");
+ }
+ }
+
+ _sts.join();
+ _cm->disable_co_trackers();
+ _sts.leave();
+
+ // we now want to allow clearing of the marking bitmap to be
+ // suspended by a collection pause.
+ _sts.join();
+ _cm->clearNextBitmap();
+ _sts.leave();
+ }
+ }
+ assert(_should_terminate, "just checking");
+
+ terminate();
+}
+
+
+void ConcurrentMarkThread::yield() {
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-yield");
+ _sts.yield("Concurrent Mark");
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-yield-end");
+}
+
+void ConcurrentMarkThread::stop() {
+ // it is ok to take late safepoints here, if needed
+ MutexLockerEx mu(Terminator_lock);
+ _should_terminate = true;
+ while (!_has_terminated) {
+ Terminator_lock->wait();
+ }
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-stop");
+}
+
+void ConcurrentMarkThread::print() {
+ gclog_or_tty->print("\"Concurrent Mark GC Thread\" ");
+ Thread::print();
+ gclog_or_tty->cr();
+}
+
+void ConcurrentMarkThread::sleepBeforeNextCycle() {
+ clear_in_progress();
+ // We join here because we don't want to do the "shouldConcurrentMark()"
+ // below while the world is otherwise stopped.
+ MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+ while (!started()) {
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-sleeping");
+ CGC_lock->wait(Mutex::_no_safepoint_check_flag);
+ }
+ set_in_progress();
+ clear_started();
+ if (TraceConcurrentMark) gclog_or_tty->print_cr("CM-starting");
+
+ return;
+}
+
+// Note: this method, although exported by the ConcurrentMarkSweepThread,
+// which is a non-JavaThread, can only be called by a JavaThread.
+// Currently this is done at vm creation time (post-vm-init) by the
+// main/Primordial (Java)Thread.
+// XXX Consider changing this in the future to allow the CMS thread
+// itself to create this thread?
+void ConcurrentMarkThread::makeSurrogateLockerThread(TRAPS) {
+ assert(_slt == NULL, "SLT already created");
+ _slt = SurrogateLockerThread::make(THREAD);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp
new file mode 100644
index 00000000000..c382778c212
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.hpp
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// The Concurrent Mark GC Thread (could be several in the future).
+// This is copied from the Concurrent Mark Sweep GC Thread
+// Still under construction.
+
+class ConcurrentMark;
+
+class ConcurrentMarkThread: public ConcurrentGCThread {
+ friend class VMStructs;
+
+ double _vtime_start; // Initial virtual time.
+ double _vtime_accum; // Accumulated virtual time.
+
+ double _vtime_mark_accum;
+ double _vtime_count_accum;
+
+ public:
+ virtual void run();
+
+ private:
+ ConcurrentMark* _cm;
+ bool _started;
+ bool _in_progress;
+
+ void sleepBeforeNextCycle();
+
+ static SurrogateLockerThread* _slt;
+
+ public:
+ // Constructor
+ ConcurrentMarkThread(ConcurrentMark* cm);
+
+ static void makeSurrogateLockerThread(TRAPS);
+ static SurrogateLockerThread* slt() { return _slt; }
+
+ // Printing
+ void print();
+
+ // Total virtual time so far.
+ double vtime_accum();
+ // Marking virtual time so far
+ double vtime_mark_accum();
+ // Counting virtual time so far.
+ double vtime_count_accum() { return _vtime_count_accum; }
+
+ ConcurrentMark* cm() { return _cm; }
+
+ void set_started() { _started = true; }
+ void clear_started() { _started = false; }
+ bool started() { return _started; }
+
+ void set_in_progress() { _in_progress = true; }
+ void clear_in_progress() { _in_progress = false; }
+ bool in_progress() { return _in_progress; }
+
+ // Yield for GC
+ void yield();
+
+ // shutdown
+ static void stop();
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.inline.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.inline.hpp
new file mode 100644
index 00000000000..b011973557f
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMarkThread.inline.hpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+ // Total virtual time so far.
+inline double ConcurrentMarkThread::vtime_accum() {
+ return _vtime_accum + _cm->all_task_accum_vtime();
+}
+
+// Marking virtual time so far
+inline double ConcurrentMarkThread::vtime_mark_accum() {
+ return _vtime_mark_accum + _cm->all_task_accum_vtime();
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp
new file mode 100644
index 00000000000..642c59cb9bf
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.cpp
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_concurrentZFThread.cpp.incl"
+
+// ======= Concurrent Zero-Fill Thread ========
+
+// The CM thread is created when the G1 garbage collector is used
+
+int ConcurrentZFThread::_region_allocs = 0;
+int ConcurrentZFThread::_sync_zfs = 0;
+int ConcurrentZFThread::_zf_waits = 0;
+int ConcurrentZFThread::_regions_filled = 0;
+
+ConcurrentZFThread::ConcurrentZFThread() :
+ ConcurrentGCThread(),
+ _co_tracker(G1ZFGroup)
+{
+ create_and_start();
+}
+
+void ConcurrentZFThread::wait_for_ZF_completed(HeapRegion* hr) {
+ assert(ZF_mon->owned_by_self(), "Precondition.");
+ note_zf_wait();
+ while (hr->zero_fill_state() == HeapRegion::ZeroFilling) {
+ ZF_mon->wait(Mutex::_no_safepoint_check_flag);
+ }
+}
+
+void ConcurrentZFThread::processHeapRegion(HeapRegion* hr) {
+ assert(!Universe::heap()->is_gc_active(),
+ "This should not happen during GC.");
+ assert(hr != NULL, "Precondition");
+ // These are unlocked reads, but if this test is successful, then no
+ // other thread will attempt this zero filling. Only a GC thread can
+ // modify the ZF state of a region whose state is zero-filling, and this
+ // should only happen while the ZF thread is locking out GC.
+ if (hr->zero_fill_state() == HeapRegion::ZeroFilling
+ && hr->zero_filler() == Thread::current()) {
+ assert(hr->top() == hr->bottom(), "better be empty!");
+ assert(!hr->isHumongous(), "Only free regions on unclean list.");
+ Copy::fill_to_words(hr->bottom(), hr->capacity()/HeapWordSize);
+ note_region_filled();
+ }
+}
+
+void ConcurrentZFThread::run() {
+ initialize_in_thread();
+ Thread* thr_self = Thread::current();
+ _vtime_start = os::elapsedVTime();
+ wait_for_universe_init();
+ _co_tracker.enable();
+ _co_tracker.start();
+
+ G1CollectedHeap* g1 = G1CollectedHeap::heap();
+ _sts.join();
+ while (!_should_terminate) {
+ _sts.leave();
+
+ {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+
+ // This local variable will hold a region being zero-filled. This
+ // region will neither be on the unclean or zero-filled lists, and
+ // will not be available for allocation; thus, we might have an
+ // allocation fail, causing a full GC, because of this, but this is a
+ // price we will pay. (In future, we might want to make the fact
+ // that there's a region being zero-filled apparent to the G1 heap,
+ // which could then wait for it in this extreme case...)
+ HeapRegion* to_fill;
+
+ while (!g1->should_zf()
+ || (to_fill = g1->pop_unclean_region_list_locked()) == NULL)
+ ZF_mon->wait(Mutex::_no_safepoint_check_flag);
+ while (to_fill->zero_fill_state() == HeapRegion::ZeroFilling)
+ ZF_mon->wait(Mutex::_no_safepoint_check_flag);
+
+ // So now to_fill is non-NULL and is not ZeroFilling. It might be
+ // Allocated or ZeroFilled. (The latter could happen if this thread
+ // starts the zero-filling of a region, but a GC intervenes and
+ // pushes new regions needing on the front of the filling on the
+ // front of the list.)
+
+ switch (to_fill->zero_fill_state()) {
+ case HeapRegion::Allocated:
+ to_fill = NULL;
+ break;
+
+ case HeapRegion::NotZeroFilled:
+ to_fill->set_zero_fill_in_progress(thr_self);
+
+ ZF_mon->unlock();
+ _sts.join();
+ processHeapRegion(to_fill);
+ _sts.leave();
+ ZF_mon->lock_without_safepoint_check();
+
+ if (to_fill->zero_fill_state() == HeapRegion::ZeroFilling
+ && to_fill->zero_filler() == thr_self) {
+ to_fill->set_zero_fill_complete();
+ (void)g1->put_free_region_on_list_locked(to_fill);
+ }
+ break;
+
+ case HeapRegion::ZeroFilled:
+ (void)g1->put_free_region_on_list_locked(to_fill);
+ break;
+
+ case HeapRegion::ZeroFilling:
+ ShouldNotReachHere();
+ break;
+ }
+ }
+ _vtime_accum = (os::elapsedVTime() - _vtime_start);
+ _sts.join();
+
+ _co_tracker.update();
+ }
+ _co_tracker.update(false);
+ _sts.leave();
+
+ assert(_should_terminate, "just checking");
+ terminate();
+}
+
+bool ConcurrentZFThread::offer_yield() {
+ if (_sts.should_yield()) {
+ _sts.yield("Concurrent ZF");
+ return true;
+ } else {
+ return false;
+ }
+}
+
+void ConcurrentZFThread::stop() {
+ // it is ok to take late safepoints here, if needed
+ MutexLockerEx mu(Terminator_lock);
+ _should_terminate = true;
+ while (!_has_terminated) {
+ Terminator_lock->wait();
+ }
+}
+
+void ConcurrentZFThread::print() {
+ gclog_or_tty->print("\"Concurrent ZF Thread\" ");
+ Thread::print();
+ gclog_or_tty->cr();
+}
+
+
+double ConcurrentZFThread::_vtime_accum;
+
+void ConcurrentZFThread::print_summary_info() {
+ gclog_or_tty->print("\nConcurrent Zero-Filling:\n");
+ gclog_or_tty->print(" Filled %d regions, used %5.2fs.\n",
+ _regions_filled,
+ vtime_accum());
+ gclog_or_tty->print(" Of %d region allocs, %d (%5.2f%%) required sync ZF,\n",
+ _region_allocs, _sync_zfs,
+ (_region_allocs > 0 ?
+ (float)_sync_zfs/(float)_region_allocs*100.0 :
+ 0.0));
+ gclog_or_tty->print(" and %d (%5.2f%%) required a ZF wait.\n",
+ _zf_waits,
+ (_region_allocs > 0 ?
+ (float)_zf_waits/(float)_region_allocs*100.0 :
+ 0.0));
+
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp b/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp
new file mode 100644
index 00000000000..9a483dbce86
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentZFThread.hpp
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// The Concurrent ZF Thread. Performs concurrent zero-filling.
+
+class ConcurrentZFThread: public ConcurrentGCThread {
+ friend class VMStructs;
+ friend class ZeroFillRegionClosure;
+
+ private:
+
+ // Zero fill the heap region.
+ void processHeapRegion(HeapRegion* r);
+
+ // Stats
+ // Allocation (protected by heap lock).
+ static int _region_allocs; // Number of regions allocated
+ static int _sync_zfs; // Synchronous zero-fills +
+ static int _zf_waits; // Wait for conc zero-fill completion.
+
+ // Number of regions CFZ thread fills.
+ static int _regions_filled;
+
+ COTracker _co_tracker;
+
+ double _vtime_start; // Initial virtual time.
+
+ // These are static because the "print_summary_info" method is, and
+ // it currently assumes there is only one ZF thread. We'll change when
+ // we need to.
+ static double _vtime_accum; // Initial virtual time.
+ static double vtime_accum() { return _vtime_accum; }
+
+ // Offer yield for GC. Returns true if yield occurred.
+ bool offer_yield();
+
+ public:
+ // Constructor
+ ConcurrentZFThread();
+
+ // Main loop.
+ virtual void run();
+
+ // Printing
+ void print();
+
+ // Waits until "r" has been zero-filled. Requires caller to hold the
+ // ZF_mon.
+ static void wait_for_ZF_completed(HeapRegion* r);
+
+ // Get or clear the current unclean region. Should be done
+ // while holding the ZF_needed_mon lock.
+
+ // shutdown
+ static void stop();
+
+ // Stats
+ static void note_region_alloc() {_region_allocs++; }
+ static void note_sync_zfs() { _sync_zfs++; }
+ static void note_zf_wait() { _zf_waits++; }
+ static void note_region_filled() { _regions_filled++; }
+
+ static void print_summary_info();
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp b/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp
new file mode 100644
index 00000000000..f3934812dd1
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp
@@ -0,0 +1,308 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+# include "incls/_precompiled.incl"
+# include "incls/_dirtyCardQueue.cpp.incl"
+
+bool DirtyCardQueue::apply_closure(CardTableEntryClosure* cl,
+ bool consume,
+ size_t worker_i) {
+ bool res = true;
+ if (_buf != NULL) {
+ res = apply_closure_to_buffer(cl, _buf, _index, _sz,
+ consume,
+ (int) worker_i);
+ if (res && consume) _index = _sz;
+ }
+ return res;
+}
+
+bool DirtyCardQueue::apply_closure_to_buffer(CardTableEntryClosure* cl,
+ void** buf,
+ size_t index, size_t sz,
+ bool consume,
+ int worker_i) {
+ if (cl == NULL) return true;
+ for (size_t i = index; i < sz; i += oopSize) {
+ int ind = byte_index_to_index((int)i);
+ jbyte* card_ptr = (jbyte*)buf[ind];
+ if (card_ptr != NULL) {
+ // Set the entry to null, so we don't do it again (via the test
+ // above) if we reconsider this buffer.
+ if (consume) buf[ind] = NULL;
+ if (!cl->do_card_ptr(card_ptr, worker_i)) return false;
+ }
+ }
+ return true;
+}
+
+#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
+#pragma warning( disable:4355 ) // 'this' : used in base member initializer list
+#endif // _MSC_VER
+
+DirtyCardQueueSet::DirtyCardQueueSet() :
+ PtrQueueSet(true /*notify_when_complete*/),
+ _closure(NULL),
+ _shared_dirty_card_queue(this, true /*perm*/),
+ _free_ids(NULL),
+ _processed_buffers_mut(0), _processed_buffers_rs_thread(0)
+{
+ _all_active = true;
+}
+
+size_t DirtyCardQueueSet::num_par_ids() {
+ return MAX2(ParallelGCThreads, (size_t)2);
+}
+
+
+void DirtyCardQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock,
+ int max_completed_queue,
+ Mutex* lock) {
+ PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue);
+ set_buffer_size(DCQBarrierQueueBufferSize);
+ set_process_completed_threshold(DCQBarrierProcessCompletedThreshold);
+
+ _shared_dirty_card_queue.set_lock(lock);
+ _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon);
+ bool b = _free_ids->claim_perm_id(0);
+ guarantee(b, "Must reserve id zero for concurrent refinement thread.");
+}
+
+void DirtyCardQueueSet::handle_zero_index_for_thread(JavaThread* t) {
+ t->dirty_card_queue().handle_zero_index();
+}
+
+void DirtyCardQueueSet::set_closure(CardTableEntryClosure* closure) {
+ _closure = closure;
+}
+
+void DirtyCardQueueSet::iterate_closure_all_threads(bool consume,
+ size_t worker_i) {
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
+ for(JavaThread* t = Threads::first(); t; t = t->next()) {
+ bool b = t->dirty_card_queue().apply_closure(_closure, consume);
+ guarantee(b, "Should not be interrupted.");
+ }
+ bool b = shared_dirty_card_queue()->apply_closure(_closure,
+ consume,
+ worker_i);
+ guarantee(b, "Should not be interrupted.");
+}
+
+bool DirtyCardQueueSet::mut_process_buffer(void** buf) {
+
+ // Used to determine if we had already claimed a par_id
+ // before entering this method.
+ bool already_claimed = false;
+
+ // We grab the current JavaThread.
+ JavaThread* thread = JavaThread::current();
+
+ // We get the the number of any par_id that this thread
+ // might have already claimed.
+ int worker_i = thread->get_claimed_par_id();
+
+ // If worker_i is not -1 then the thread has already claimed
+ // a par_id. We make note of it using the already_claimed value
+ if (worker_i != -1) {
+ already_claimed = true;
+ } else {
+
+ // Otherwise we need to claim a par id
+ worker_i = _free_ids->claim_par_id();
+
+ // And store the par_id value in the thread
+ thread->set_claimed_par_id(worker_i);
+ }
+
+ bool b = false;
+ if (worker_i != -1) {
+ b = DirtyCardQueue::apply_closure_to_buffer(_closure, buf, 0,
+ _sz, true, worker_i);
+ if (b) Atomic::inc(&_processed_buffers_mut);
+
+ // If we had not claimed an id before entering the method
+ // then we must release the id.
+ if (!already_claimed) {
+
+ // we release the id
+ _free_ids->release_par_id(worker_i);
+
+ // and set the claimed_id in the thread to -1
+ thread->set_claimed_par_id(-1);
+ }
+ }
+ return b;
+}
+
+DirtyCardQueueSet::CompletedBufferNode*
+DirtyCardQueueSet::get_completed_buffer_lock(int stop_at) {
+ CompletedBufferNode* nd = NULL;
+ MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
+
+ if ((int)_n_completed_buffers <= stop_at) {
+ _process_completed = false;
+ return NULL;
+ }
+
+ if (_completed_buffers_head != NULL) {
+ nd = _completed_buffers_head;
+ _completed_buffers_head = nd->next;
+ if (_completed_buffers_head == NULL)
+ _completed_buffers_tail = NULL;
+ _n_completed_buffers--;
+ }
+ debug_only(assert_completed_buffer_list_len_correct_locked());
+ return nd;
+}
+
+// We only do this in contexts where there is no concurrent enqueueing.
+DirtyCardQueueSet::CompletedBufferNode*
+DirtyCardQueueSet::get_completed_buffer_CAS() {
+ CompletedBufferNode* nd = _completed_buffers_head;
+
+ while (nd != NULL) {
+ CompletedBufferNode* next = nd->next;
+ CompletedBufferNode* result =
+ (CompletedBufferNode*)Atomic::cmpxchg_ptr(next,
+ &_completed_buffers_head,
+ nd);
+ if (result == nd) {
+ return result;
+ } else {
+ nd = _completed_buffers_head;
+ }
+ }
+ assert(_completed_buffers_head == NULL, "Loop post");
+ _completed_buffers_tail = NULL;
+ return NULL;
+}
+
+bool DirtyCardQueueSet::
+apply_closure_to_completed_buffer_helper(int worker_i,
+ CompletedBufferNode* nd) {
+ if (nd != NULL) {
+ bool b =
+ DirtyCardQueue::apply_closure_to_buffer(_closure, nd->buf,
+ nd->index, _sz,
+ true, worker_i);
+ void** buf = nd->buf;
+ size_t index = nd->index;
+ delete nd;
+ if (b) {
+ deallocate_buffer(buf);
+ return true; // In normal case, go on to next buffer.
+ } else {
+ enqueue_complete_buffer(buf, index, true);
+ return false;
+ }
+ } else {
+ return false;
+ }
+}
+
+bool DirtyCardQueueSet::apply_closure_to_completed_buffer(int worker_i,
+ int stop_at,
+ bool with_CAS)
+{
+ CompletedBufferNode* nd = NULL;
+ if (with_CAS) {
+ guarantee(stop_at == 0, "Precondition");
+ nd = get_completed_buffer_CAS();
+ } else {
+ nd = get_completed_buffer_lock(stop_at);
+ }
+ bool res = apply_closure_to_completed_buffer_helper(worker_i, nd);
+ if (res) _processed_buffers_rs_thread++;
+ return res;
+}
+
+void DirtyCardQueueSet::apply_closure_to_all_completed_buffers() {
+ CompletedBufferNode* nd = _completed_buffers_head;
+ while (nd != NULL) {
+ bool b =
+ DirtyCardQueue::apply_closure_to_buffer(_closure, nd->buf, 0, _sz,
+ false);
+ guarantee(b, "Should not stop early.");
+ nd = nd->next;
+ }
+}
+
+void DirtyCardQueueSet::abandon_logs() {
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
+ CompletedBufferNode* buffers_to_delete = NULL;
+ {
+ MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
+ while (_completed_buffers_head != NULL) {
+ CompletedBufferNode* nd = _completed_buffers_head;
+ _completed_buffers_head = nd->next;
+ nd->next = buffers_to_delete;
+ buffers_to_delete = nd;
+ }
+ _n_completed_buffers = 0;
+ _completed_buffers_tail = NULL;
+ debug_only(assert_completed_buffer_list_len_correct_locked());
+ }
+ while (buffers_to_delete != NULL) {
+ CompletedBufferNode* nd = buffers_to_delete;
+ buffers_to_delete = nd->next;
+ deallocate_buffer(nd->buf);
+ delete nd;
+ }
+ // Since abandon is done only at safepoints, we can safely manipulate
+ // these queues.
+ for (JavaThread* t = Threads::first(); t; t = t->next()) {
+ t->dirty_card_queue().reset();
+ }
+ shared_dirty_card_queue()->reset();
+}
+
+
+void DirtyCardQueueSet::concatenate_logs() {
+ // Iterate over all the threads, if we find a partial log add it to
+ // the global list of logs. Temporarily turn off the limit on the number
+ // of outstanding buffers.
+ int save_max_completed_queue = _max_completed_queue;
+ _max_completed_queue = max_jint;
+ assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
+ for (JavaThread* t = Threads::first(); t; t = t->next()) {
+ DirtyCardQueue& dcq = t->dirty_card_queue();
+ if (dcq.size() != 0) {
+ void **buf = t->dirty_card_queue().get_buf();
+ // We must NULL out the unused entries, then enqueue.
+ for (size_t i = 0; i < t->dirty_card_queue().get_index(); i += oopSize) {
+ buf[PtrQueue::byte_index_to_index((int)i)] = NULL;
+ }
+ enqueue_complete_buffer(dcq.get_buf(), dcq.get_index());
+ dcq.reinitialize();
+ }
+ }
+ if (_shared_dirty_card_queue.size() != 0) {
+ enqueue_complete_buffer(_shared_dirty_card_queue.get_buf(),
+ _shared_dirty_card_queue.get_index());
+ _shared_dirty_card_queue.reinitialize();
+ }
+ // Restore the completed buffer queue limit.
+ _max_completed_queue = save_max_completed_queue;
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp b/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp
new file mode 100644
index 00000000000..86876fd949d
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp
@@ -0,0 +1,152 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+class FreeIdSet;
+
+// A closure class for processing card table entries. Note that we don't
+// require these closure objects to be stack-allocated.
+class CardTableEntryClosure: public CHeapObj {
+public:
+ // Process the card whose card table entry is "card_ptr". If returns
+ // "false", terminate the iteration early.
+ virtual bool do_card_ptr(jbyte* card_ptr, int worker_i = 0) = 0;
+};
+
+// A ptrQueue whose elements are "oops", pointers to object heads.
+class DirtyCardQueue: public PtrQueue {
+public:
+ DirtyCardQueue(PtrQueueSet* qset_, bool perm = false) :
+ PtrQueue(qset_, perm)
+ {
+ // Dirty card queues are always active.
+ _active = true;
+ }
+ // Apply the closure to all elements, and reset the index to make the
+ // buffer empty. If a closure application returns "false", return
+ // "false" immediately, halting the iteration. If "consume" is true,
+ // deletes processed entries from logs.
+ bool apply_closure(CardTableEntryClosure* cl,
+ bool consume = true,
+ size_t worker_i = 0);
+
+ // Apply the closure to all elements of "buf", down to "index"
+ // (inclusive.) If returns "false", then a closure application returned
+ // "false", and we return immediately. If "consume" is true, entries are
+ // set to NULL as they are processed, so they will not be processed again
+ // later.
+ static bool apply_closure_to_buffer(CardTableEntryClosure* cl,
+ void** buf, size_t index, size_t sz,
+ bool consume = true,
+ int worker_i = 0);
+ void **get_buf() { return _buf;}
+ void set_buf(void **buf) {_buf = buf;}
+ size_t get_index() { return _index;}
+ void reinitialize() { _buf = 0; _sz = 0; _index = 0;}
+};
+
+
+
+class DirtyCardQueueSet: public PtrQueueSet {
+ CardTableEntryClosure* _closure;
+
+ DirtyCardQueue _shared_dirty_card_queue;
+
+ // Override.
+ bool mut_process_buffer(void** buf);
+
+ // Protected by the _cbl_mon.
+ FreeIdSet* _free_ids;
+
+ // The number of completed buffers processed by mutator and rs thread,
+ // respectively.
+ jint _processed_buffers_mut;
+ jint _processed_buffers_rs_thread;
+
+public:
+ DirtyCardQueueSet();
+
+ void initialize(Monitor* cbl_mon, Mutex* fl_lock,
+ int max_completed_queue = 0,
+ Mutex* lock = NULL);
+
+ // The number of parallel ids that can be claimed to allow collector or
+ // mutator threads to do card-processing work.
+ static size_t num_par_ids();
+
+ static void handle_zero_index_for_thread(JavaThread* t);
+
+ // Register "blk" as "the closure" for all queues. Only one such closure
+ // is allowed. The "apply_closure_to_completed_buffer" method will apply
+ // this closure to a completed buffer, and "iterate_closure_all_threads"
+ // applies it to partially-filled buffers (the latter should only be done
+ // with the world stopped).
+ void set_closure(CardTableEntryClosure* closure);
+
+ // If there is a registered closure for buffers, apply it to all entries
+ // in all currently-active buffers. This should only be applied at a
+ // safepoint. (Currently must not be called in parallel; this should
+ // change in the future.) If "consume" is true, processed entries are
+ // discarded.
+ void iterate_closure_all_threads(bool consume = true,
+ size_t worker_i = 0);
+
+ // If there exists some completed buffer, pop it, then apply the
+ // registered closure to all its elements, nulling out those elements
+ // processed. If all elements are processed, returns "true". If no
+ // completed buffers exist, returns false. If a completed buffer exists,
+ // but is only partially completed before a "yield" happens, the
+ // partially completed buffer (with its processed elements set to NULL)
+ // is returned to the completed buffer set, and this call returns false.
+ bool apply_closure_to_completed_buffer(int worker_i = 0,
+ int stop_at = 0,
+ bool with_CAS = false);
+ bool apply_closure_to_completed_buffer_helper(int worker_i,
+ CompletedBufferNode* nd);
+
+ CompletedBufferNode* get_completed_buffer_CAS();
+ CompletedBufferNode* get_completed_buffer_lock(int stop_at);
+ // Applies the current closure to all completed buffers,
+ // non-consumptively.
+ void apply_closure_to_all_completed_buffers();
+
+ DirtyCardQueue* shared_dirty_card_queue() {
+ return &_shared_dirty_card_queue;
+ }
+
+ // If a full collection is happening, reset partial logs, and ignore
+ // completed ones: the full collection will make them all irrelevant.
+ void abandon_logs();
+
+ // If any threads have partial logs, add them to the global list of logs.
+ void concatenate_logs();
+ void clear_n_completed_buffers() { _n_completed_buffers = 0;}
+
+ jint processed_buffers_mut() {
+ return _processed_buffers_mut;
+ }
+ jint processed_buffers_rs_thread() {
+ return _processed_buffers_rs_thread;
+ }
+
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp
new file mode 100644
index 00000000000..eddbf86d999
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp
@@ -0,0 +1,628 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_g1BlockOffsetTable.cpp.incl"
+
+//////////////////////////////////////////////////////////////////////
+// G1BlockOffsetSharedArray
+//////////////////////////////////////////////////////////////////////
+
+G1BlockOffsetSharedArray::G1BlockOffsetSharedArray(MemRegion reserved,
+ size_t init_word_size) :
+ _reserved(reserved), _end(NULL)
+{
+ size_t size = compute_size(reserved.word_size());
+ ReservedSpace rs(ReservedSpace::allocation_align_size_up(size));
+ if (!rs.is_reserved()) {
+ vm_exit_during_initialization("Could not reserve enough space for heap offset array");
+ }
+ if (!_vs.initialize(rs, 0)) {
+ vm_exit_during_initialization("Could not reserve enough space for heap offset array");
+ }
+ _offset_array = (u_char*)_vs.low_boundary();
+ resize(init_word_size);
+ if (TraceBlockOffsetTable) {
+ gclog_or_tty->print_cr("G1BlockOffsetSharedArray::G1BlockOffsetSharedArray: ");
+ gclog_or_tty->print_cr(" "
+ " rs.base(): " INTPTR_FORMAT
+ " rs.size(): " INTPTR_FORMAT
+ " rs end(): " INTPTR_FORMAT,
+ rs.base(), rs.size(), rs.base() + rs.size());
+ gclog_or_tty->print_cr(" "
+ " _vs.low_boundary(): " INTPTR_FORMAT
+ " _vs.high_boundary(): " INTPTR_FORMAT,
+ _vs.low_boundary(),
+ _vs.high_boundary());
+ }
+}
+
+void G1BlockOffsetSharedArray::resize(size_t new_word_size) {
+ assert(new_word_size <= _reserved.word_size(), "Resize larger than reserved");
+ size_t new_size = compute_size(new_word_size);
+ size_t old_size = _vs.committed_size();
+ size_t delta;
+ char* high = _vs.high();
+ _end = _reserved.start() + new_word_size;
+ if (new_size > old_size) {
+ delta = ReservedSpace::page_align_size_up(new_size - old_size);
+ assert(delta > 0, "just checking");
+ if (!_vs.expand_by(delta)) {
+ // Do better than this for Merlin
+ vm_exit_out_of_memory(delta, "offset table expansion");
+ }
+ assert(_vs.high() == high + delta, "invalid expansion");
+ // Initialization of the contents is left to the
+ // G1BlockOffsetArray that uses it.
+ } else {
+ delta = ReservedSpace::page_align_size_down(old_size - new_size);
+ if (delta == 0) return;
+ _vs.shrink_by(delta);
+ assert(_vs.high() == high - delta, "invalid expansion");
+ }
+}
+
+bool G1BlockOffsetSharedArray::is_card_boundary(HeapWord* p) const {
+ assert(p >= _reserved.start(), "just checking");
+ size_t delta = pointer_delta(p, _reserved.start());
+ return (delta & right_n_bits(LogN_words)) == (size_t)NoBits;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+// G1BlockOffsetArray
+//////////////////////////////////////////////////////////////////////
+
+G1BlockOffsetArray::G1BlockOffsetArray(G1BlockOffsetSharedArray* array,
+ MemRegion mr, bool init_to_zero) :
+ G1BlockOffsetTable(mr.start(), mr.end()),
+ _unallocated_block(_bottom),
+ _array(array), _csp(NULL),
+ _init_to_zero(init_to_zero) {
+ assert(_bottom <= _end, "arguments out of order");
+ if (!_init_to_zero) {
+ // initialize cards to point back to mr.start()
+ set_remainder_to_point_to_start(mr.start() + N_words, mr.end());
+ _array->set_offset_array(0, 0); // set first card to 0
+ }
+}
+
+void G1BlockOffsetArray::set_space(Space* sp) {
+ _sp = sp;
+ _csp = sp->toContiguousSpace();
+}
+
+// The arguments follow the normal convention of denoting
+// a right-open interval: [start, end)
+void
+G1BlockOffsetArray:: set_remainder_to_point_to_start(HeapWord* start, HeapWord* end) {
+
+ if (start >= end) {
+ // The start address is equal to the end address (or to
+ // the right of the end address) so there are not cards
+ // that need to be updated..
+ return;
+ }
+
+ // Write the backskip value for each region.
+ //
+ // offset
+ // card 2nd 3rd
+ // | +- 1st | |
+ // v v v v
+ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+-
+ // |x|0|0|0|0|0|0|0|1|1|1|1|1|1| ... |1|1|1|1|2|2|2|2|2|2| ...
+ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+-+-+-
+ // 11 19 75
+ // 12
+ //
+ // offset card is the card that points to the start of an object
+ // x - offset value of offset card
+ // 1st - start of first logarithmic region
+ // 0 corresponds to logarithmic value N_words + 0 and 2**(3 * 0) = 1
+ // 2nd - start of second logarithmic region
+ // 1 corresponds to logarithmic value N_words + 1 and 2**(3 * 1) = 8
+ // 3rd - start of third logarithmic region
+ // 2 corresponds to logarithmic value N_words + 2 and 2**(3 * 2) = 64
+ //
+ // integer below the block offset entry is an example of
+ // the index of the entry
+ //
+ // Given an address,
+ // Find the index for the address
+ // Find the block offset table entry
+ // Convert the entry to a back slide
+ // (e.g., with today's, offset = 0x81 =>
+ // back slip = 2**(3*(0x81 - N_words)) = 2**3) = 8
+ // Move back N (e.g., 8) entries and repeat with the
+ // value of the new entry
+ //
+ size_t start_card = _array->index_for(start);
+ size_t end_card = _array->index_for(end-1);
+ assert(start ==_array->address_for_index(start_card), "Precondition");
+ assert(end ==_array->address_for_index(end_card)+N_words, "Precondition");
+ set_remainder_to_point_to_start_incl(start_card, end_card); // closed interval
+}
+
+// Unlike the normal convention in this code, the argument here denotes
+// a closed, inclusive interval: [start_card, end_card], cf set_remainder_to_point_to_start()
+// above.
+void
+G1BlockOffsetArray::set_remainder_to_point_to_start_incl(size_t start_card, size_t end_card) {
+ if (start_card > end_card) {
+ return;
+ }
+ assert(start_card > _array->index_for(_bottom), "Cannot be first card");
+ assert(_array->offset_array(start_card-1) <= N_words,
+ "Offset card has an unexpected value");
+ size_t start_card_for_region = start_card;
+ u_char offset = max_jubyte;
+ for (int i = 0; i < BlockOffsetArray::N_powers; i++) {
+ // -1 so that the the card with the actual offset is counted. Another -1
+ // so that the reach ends in this region and not at the start
+ // of the next.
+ size_t reach = start_card - 1 + (BlockOffsetArray::power_to_cards_back(i+1) - 1);
+ offset = N_words + i;
+ if (reach >= end_card) {
+ _array->set_offset_array(start_card_for_region, end_card, offset);
+ start_card_for_region = reach + 1;
+ break;
+ }
+ _array->set_offset_array(start_card_for_region, reach, offset);
+ start_card_for_region = reach + 1;
+ }
+ assert(start_card_for_region > end_card, "Sanity check");
+ DEBUG_ONLY(check_all_cards(start_card, end_card);)
+}
+
+// The block [blk_start, blk_end) has been allocated;
+// adjust the block offset table to represent this information;
+// right-open interval: [blk_start, blk_end)
+void
+G1BlockOffsetArray::alloc_block(HeapWord* blk_start, HeapWord* blk_end) {
+ mark_block(blk_start, blk_end);
+ allocated(blk_start, blk_end);
+}
+
+// Adjust BOT to show that a previously whole block has been split
+// into two.
+void G1BlockOffsetArray::split_block(HeapWord* blk, size_t blk_size,
+ size_t left_blk_size) {
+ // Verify that the BOT shows [blk, blk + blk_size) to be one block.
+ verify_single_block(blk, blk_size);
+ // Update the BOT to indicate that [blk + left_blk_size, blk + blk_size)
+ // is one single block.
+ mark_block(blk + left_blk_size, blk + blk_size);
+}
+
+
+// Action_mark - update the BOT for the block [blk_start, blk_end).
+// Current typical use is for splitting a block.
+// Action_single - udpate the BOT for an allocation.
+// Action_verify - BOT verification.
+void G1BlockOffsetArray::do_block_internal(HeapWord* blk_start,
+ HeapWord* blk_end,
+ Action action) {
+ assert(Universe::heap()->is_in_reserved(blk_start),
+ "reference must be into the heap");
+ assert(Universe::heap()->is_in_reserved(blk_end-1),
+ "limit must be within the heap");
+ // This is optimized to make the test fast, assuming we only rarely
+ // cross boundaries.
+ uintptr_t end_ui = (uintptr_t)(blk_end - 1);
+ uintptr_t start_ui = (uintptr_t)blk_start;
+ // Calculate the last card boundary preceding end of blk
+ intptr_t boundary_before_end = (intptr_t)end_ui;
+ clear_bits(boundary_before_end, right_n_bits(LogN));
+ if (start_ui <= (uintptr_t)boundary_before_end) {
+ // blk starts at or crosses a boundary
+ // Calculate index of card on which blk begins
+ size_t start_index = _array->index_for(blk_start);
+ // Index of card on which blk ends
+ size_t end_index = _array->index_for(blk_end - 1);
+ // Start address of card on which blk begins
+ HeapWord* boundary = _array->address_for_index(start_index);
+ assert(boundary <= blk_start, "blk should start at or after boundary");
+ if (blk_start != boundary) {
+ // blk starts strictly after boundary
+ // adjust card boundary and start_index forward to next card
+ boundary += N_words;
+ start_index++;
+ }
+ assert(start_index <= end_index, "monotonicity of index_for()");
+ assert(boundary <= (HeapWord*)boundary_before_end, "tautology");
+ switch (action) {
+ case Action_mark: {
+ if (init_to_zero()) {
+ _array->set_offset_array(start_index, boundary, blk_start);
+ break;
+ } // Else fall through to the next case
+ }
+ case Action_single: {
+ _array->set_offset_array(start_index, boundary, blk_start);
+ // We have finished marking the "offset card". We need to now
+ // mark the subsequent cards that this blk spans.
+ if (start_index < end_index) {
+ HeapWord* rem_st = _array->address_for_index(start_index) + N_words;
+ HeapWord* rem_end = _array->address_for_index(end_index) + N_words;
+ set_remainder_to_point_to_start(rem_st, rem_end);
+ }
+ break;
+ }
+ case Action_check: {
+ _array->check_offset_array(start_index, boundary, blk_start);
+ // We have finished checking the "offset card". We need to now
+ // check the subsequent cards that this blk spans.
+ check_all_cards(start_index + 1, end_index);
+ break;
+ }
+ default:
+ ShouldNotReachHere();
+ }
+ }
+}
+
+// The card-interval [start_card, end_card] is a closed interval; this
+// is an expensive check -- use with care and only under protection of
+// suitable flag.
+void G1BlockOffsetArray::check_all_cards(size_t start_card, size_t end_card) const {
+
+ if (end_card < start_card) {
+ return;
+ }
+ guarantee(_array->offset_array(start_card) == N_words, "Wrong value in second card");
+ for (size_t c = start_card + 1; c <= end_card; c++ /* yeah! */) {
+ u_char entry = _array->offset_array(c);
+ if (c - start_card > BlockOffsetArray::power_to_cards_back(1)) {
+ guarantee(entry > N_words, "Should be in logarithmic region");
+ }
+ size_t backskip = BlockOffsetArray::entry_to_cards_back(entry);
+ size_t landing_card = c - backskip;
+ guarantee(landing_card >= (start_card - 1), "Inv");
+ if (landing_card >= start_card) {
+ guarantee(_array->offset_array(landing_card) <= entry, "monotonicity");
+ } else {
+ guarantee(landing_card == start_card - 1, "Tautology");
+ guarantee(_array->offset_array(landing_card) <= N_words, "Offset value");
+ }
+ }
+}
+
+// The range [blk_start, blk_end) represents a single contiguous block
+// of storage; modify the block offset table to represent this
+// information; Right-open interval: [blk_start, blk_end)
+// NOTE: this method does _not_ adjust _unallocated_block.
+void
+G1BlockOffsetArray::single_block(HeapWord* blk_start, HeapWord* blk_end) {
+ do_block_internal(blk_start, blk_end, Action_single);
+}
+
+// Mark the BOT such that if [blk_start, blk_end) straddles a card
+// boundary, the card following the first such boundary is marked
+// with the appropriate offset.
+// NOTE: this method does _not_ adjust _unallocated_block or
+// any cards subsequent to the first one.
+void
+G1BlockOffsetArray::mark_block(HeapWord* blk_start, HeapWord* blk_end) {
+ do_block_internal(blk_start, blk_end, Action_mark);
+}
+
+void G1BlockOffsetArray::join_blocks(HeapWord* blk1, HeapWord* blk2) {
+ HeapWord* blk1_start = Universe::heap()->block_start(blk1);
+ HeapWord* blk2_start = Universe::heap()->block_start(blk2);
+ assert(blk1 == blk1_start && blk2 == blk2_start,
+ "Must be block starts.");
+ assert(blk1 + _sp->block_size(blk1) == blk2, "Must be contiguous.");
+ size_t blk1_start_index = _array->index_for(blk1);
+ size_t blk2_start_index = _array->index_for(blk2);
+ assert(blk1_start_index <= blk2_start_index, "sanity");
+ HeapWord* blk2_card_start = _array->address_for_index(blk2_start_index);
+ if (blk2 == blk2_card_start) {
+ // blk2 starts a card. Does blk1 start on the prevous card, or futher
+ // back?
+ assert(blk1_start_index < blk2_start_index, "must be lower card.");
+ if (blk1_start_index + 1 == blk2_start_index) {
+ // previous card; new value for blk2 card is size of blk1.
+ _array->set_offset_array(blk2_start_index, (u_char) _sp->block_size(blk1));
+ } else {
+ // Earlier card; go back a card.
+ _array->set_offset_array(blk2_start_index, N_words);
+ }
+ } else {
+ // blk2 does not start a card. Does it cross a card? If not, nothing
+ // to do.
+ size_t blk2_end_index =
+ _array->index_for(blk2 + _sp->block_size(blk2) - 1);
+ assert(blk2_end_index >= blk2_start_index, "sanity");
+ if (blk2_end_index > blk2_start_index) {
+ // Yes, it crosses a card. The value for the next card must change.
+ if (blk1_start_index + 1 == blk2_start_index) {
+ // previous card; new value for second blk2 card is size of blk1.
+ _array->set_offset_array(blk2_start_index + 1,
+ (u_char) _sp->block_size(blk1));
+ } else {
+ // Earlier card; go back a card.
+ _array->set_offset_array(blk2_start_index + 1, N_words);
+ }
+ }
+ }
+}
+
+HeapWord* G1BlockOffsetArray::block_start_unsafe(const void* addr) {
+ assert(_bottom <= addr && addr < _end,
+ "addr must be covered by this Array");
+ // Must read this exactly once because it can be modified by parallel
+ // allocation.
+ HeapWord* ub = _unallocated_block;
+ if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) {
+ assert(ub < _end, "tautology (see above)");
+ return ub;
+ }
+ // Otherwise, find the block start using the table.
+ HeapWord* q = block_at_or_preceding(addr, false, 0);
+ return forward_to_block_containing_addr(q, addr);
+}
+
+// This duplicates a little code from the above: unavoidable.
+HeapWord*
+G1BlockOffsetArray::block_start_unsafe_const(const void* addr) const {
+ assert(_bottom <= addr && addr < _end,
+ "addr must be covered by this Array");
+ // Must read this exactly once because it can be modified by parallel
+ // allocation.
+ HeapWord* ub = _unallocated_block;
+ if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) {
+ assert(ub < _end, "tautology (see above)");
+ return ub;
+ }
+ // Otherwise, find the block start using the table.
+ HeapWord* q = block_at_or_preceding(addr, false, 0);
+ HeapWord* n = q + _sp->block_size(q);
+ return forward_to_block_containing_addr_const(q, n, addr);
+}
+
+
+HeapWord*
+G1BlockOffsetArray::forward_to_block_containing_addr_slow(HeapWord* q,
+ HeapWord* n,
+ const void* addr) {
+ // We're not in the normal case. We need to handle an important subcase
+ // here: LAB allocation. An allocation previously recorded in the
+ // offset table was actually a lab allocation, and was divided into
+ // several objects subsequently. Fix this situation as we answer the
+ // query, by updating entries as we cross them.
+
+ // If the fist object's end q is at the card boundary. Start refining
+ // with the corresponding card (the value of the entry will be basically
+ // set to 0). If the object crosses the boundary -- start from the next card.
+ size_t next_index = _array->index_for(n) + !_array->is_card_boundary(n);
+ HeapWord* next_boundary = _array->address_for_index(next_index);
+ if (csp() != NULL) {
+ if (addr >= csp()->top()) return csp()->top();
+ while (next_boundary < addr) {
+ while (n <= next_boundary) {
+ q = n;
+ oop obj = oop(q);
+ if (obj->klass() == NULL) return q;
+ n += obj->size();
+ }
+ assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
+ // [q, n) is the block that crosses the boundary.
+ alloc_block_work2(&next_boundary, &next_index, q, n);
+ }
+ } else {
+ while (next_boundary < addr) {
+ while (n <= next_boundary) {
+ q = n;
+ oop obj = oop(q);
+ if (obj->klass() == NULL) return q;
+ n += _sp->block_size(q);
+ }
+ assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
+ // [q, n) is the block that crosses the boundary.
+ alloc_block_work2(&next_boundary, &next_index, q, n);
+ }
+ }
+ return forward_to_block_containing_addr_const(q, n, addr);
+}
+
+HeapWord* G1BlockOffsetArray::block_start_careful(const void* addr) const {
+ assert(_array->offset_array(0) == 0, "objects can't cross covered areas");
+
+ assert(_bottom <= addr && addr < _end,
+ "addr must be covered by this Array");
+ // Must read this exactly once because it can be modified by parallel
+ // allocation.
+ HeapWord* ub = _unallocated_block;
+ if (BlockOffsetArrayUseUnallocatedBlock && addr >= ub) {
+ assert(ub < _end, "tautology (see above)");
+ return ub;
+ }
+
+ // Otherwise, find the block start using the table, but taking
+ // care (cf block_start_unsafe() above) not to parse any objects/blocks
+ // on the cards themsleves.
+ size_t index = _array->index_for(addr);
+ assert(_array->address_for_index(index) == addr,
+ "arg should be start of card");
+
+ HeapWord* q = (HeapWord*)addr;
+ uint offset;
+ do {
+ offset = _array->offset_array(index--);
+ q -= offset;
+ } while (offset == N_words);
+ assert(q <= addr, "block start should be to left of arg");
+ return q;
+}
+
+// Note that the committed size of the covered space may have changed,
+// so the table size might also wish to change.
+void G1BlockOffsetArray::resize(size_t new_word_size) {
+ HeapWord* new_end = _bottom + new_word_size;
+ if (_end < new_end && !init_to_zero()) {
+ // verify that the old and new boundaries are also card boundaries
+ assert(_array->is_card_boundary(_end),
+ "_end not a card boundary");
+ assert(_array->is_card_boundary(new_end),
+ "new _end would not be a card boundary");
+ // set all the newly added cards
+ _array->set_offset_array(_end, new_end, N_words);
+ }
+ _end = new_end; // update _end
+}
+
+void G1BlockOffsetArray::set_region(MemRegion mr) {
+ _bottom = mr.start();
+ _end = mr.end();
+}
+
+//
+// threshold_
+// | _index_
+// v v
+// +-------+-------+-------+-------+-------+
+// | i-1 | i | i+1 | i+2 | i+3 |
+// +-------+-------+-------+-------+-------+
+// ( ^ ]
+// block-start
+//
+void G1BlockOffsetArray::alloc_block_work2(HeapWord** threshold_, size_t* index_,
+ HeapWord* blk_start, HeapWord* blk_end) {
+ // For efficiency, do copy-in/copy-out.
+ HeapWord* threshold = *threshold_;
+ size_t index = *index_;
+
+ assert(blk_start != NULL && blk_end > blk_start,
+ "phantom block");
+ assert(blk_end > threshold, "should be past threshold");
+ assert(blk_start <= threshold, "blk_start should be at or before threshold")
+ assert(pointer_delta(threshold, blk_start) <= N_words,
+ "offset should be <= BlockOffsetSharedArray::N");
+ assert(Universe::heap()->is_in_reserved(blk_start),
+ "reference must be into the heap");
+ assert(Universe::heap()->is_in_reserved(blk_end-1),
+ "limit must be within the heap");
+ assert(threshold == _array->_reserved.start() + index*N_words,
+ "index must agree with threshold");
+
+ DEBUG_ONLY(size_t orig_index = index;)
+
+ // Mark the card that holds the offset into the block. Note
+ // that _next_offset_index and _next_offset_threshold are not
+ // updated until the end of this method.
+ _array->set_offset_array(index, threshold, blk_start);
+
+ // We need to now mark the subsequent cards that this blk spans.
+
+ // Index of card on which blk ends.
+ size_t end_index = _array->index_for(blk_end - 1);
+
+ // Are there more cards left to be updated?
+ if (index + 1 <= end_index) {
+ HeapWord* rem_st = _array->address_for_index(index + 1);
+ // Calculate rem_end this way because end_index
+ // may be the last valid index in the covered region.
+ HeapWord* rem_end = _array->address_for_index(end_index) + N_words;
+ set_remainder_to_point_to_start(rem_st, rem_end);
+ }
+
+ index = end_index + 1;
+ // Calculate threshold_ this way because end_index
+ // may be the last valid index in the covered region.
+ threshold = _array->address_for_index(end_index) + N_words;
+ assert(threshold >= blk_end, "Incorrect offset threshold");
+
+ // index_ and threshold_ updated here.
+ *threshold_ = threshold;
+ *index_ = index;
+
+#ifdef ASSERT
+ // The offset can be 0 if the block starts on a boundary. That
+ // is checked by an assertion above.
+ size_t start_index = _array->index_for(blk_start);
+ HeapWord* boundary = _array->address_for_index(start_index);
+ assert((_array->offset_array(orig_index) == 0 &&
+ blk_start == boundary) ||
+ (_array->offset_array(orig_index) > 0 &&
+ _array->offset_array(orig_index) <= N_words),
+ "offset array should have been set");
+ for (size_t j = orig_index + 1; j <= end_index; j++) {
+ assert(_array->offset_array(j) > 0 &&
+ _array->offset_array(j) <=
+ (u_char) (N_words+BlockOffsetArray::N_powers-1),
+ "offset array should have been set");
+ }
+#endif
+}
+
+//////////////////////////////////////////////////////////////////////
+// G1BlockOffsetArrayContigSpace
+//////////////////////////////////////////////////////////////////////
+
+HeapWord*
+G1BlockOffsetArrayContigSpace::block_start_unsafe(const void* addr) {
+ assert(_bottom <= addr && addr < _end,
+ "addr must be covered by this Array");
+ HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1);
+ return forward_to_block_containing_addr(q, addr);
+}
+
+HeapWord*
+G1BlockOffsetArrayContigSpace::
+block_start_unsafe_const(const void* addr) const {
+ assert(_bottom <= addr && addr < _end,
+ "addr must be covered by this Array");
+ HeapWord* q = block_at_or_preceding(addr, true, _next_offset_index-1);
+ HeapWord* n = q + _sp->block_size(q);
+ return forward_to_block_containing_addr_const(q, n, addr);
+}
+
+G1BlockOffsetArrayContigSpace::
+G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array,
+ MemRegion mr) :
+ G1BlockOffsetArray(array, mr, true)
+{
+ _next_offset_threshold = NULL;
+ _next_offset_index = 0;
+}
+
+HeapWord* G1BlockOffsetArrayContigSpace::initialize_threshold() {
+ assert(!Universe::heap()->is_in_reserved(_array->_offset_array),
+ "just checking");
+ _next_offset_index = _array->index_for(_bottom);
+ _next_offset_index++;
+ _next_offset_threshold =
+ _array->address_for_index(_next_offset_index);
+ return _next_offset_threshold;
+}
+
+void G1BlockOffsetArrayContigSpace::zero_bottom_entry() {
+ assert(!Universe::heap()->is_in_reserved(_array->_offset_array),
+ "just checking");
+ size_t bottom_index = _array->index_for(_bottom);
+ assert(_array->address_for_index(bottom_index) == _bottom,
+ "Precondition of call");
+ _array->set_offset_array(bottom_index, 0);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp
new file mode 100644
index 00000000000..8c72f768933
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.hpp
@@ -0,0 +1,487 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// The CollectedHeap type requires subtypes to implement a method
+// "block_start". For some subtypes, notably generational
+// systems using card-table-based write barriers, the efficiency of this
+// operation may be important. Implementations of the "BlockOffsetArray"
+// class may be useful in providing such efficient implementations.
+//
+// While generally mirroring the structure of the BOT for GenCollectedHeap,
+// the following types are tailored more towards G1's uses; these should,
+// however, be merged back into a common BOT to avoid code duplication
+// and reduce maintenance overhead.
+//
+// G1BlockOffsetTable (abstract)
+// -- G1BlockOffsetArray (uses G1BlockOffsetSharedArray)
+// -- G1BlockOffsetArrayContigSpace
+//
+// A main impediment to the consolidation of this code might be the
+// effect of making some of the block_start*() calls non-const as
+// below. Whether that might adversely affect performance optimizations
+// that compilers might normally perform in the case of non-G1
+// collectors needs to be carefully investigated prior to any such
+// consolidation.
+
+// Forward declarations
+class ContiguousSpace;
+class G1BlockOffsetSharedArray;
+
+class G1BlockOffsetTable VALUE_OBJ_CLASS_SPEC {
+ friend class VMStructs;
+protected:
+ // These members describe the region covered by the table.
+
+ // The space this table is covering.
+ HeapWord* _bottom; // == reserved.start
+ HeapWord* _end; // End of currently allocated region.
+
+public:
+ // Initialize the table to cover the given space.
+ // The contents of the initial table are undefined.
+ G1BlockOffsetTable(HeapWord* bottom, HeapWord* end) :
+ _bottom(bottom), _end(end)
+ {
+ assert(_bottom <= _end, "arguments out of order");
+ }
+
+ // Note that the committed size of the covered space may have changed,
+ // so the table size might also wish to change.
+ virtual void resize(size_t new_word_size) = 0;
+
+ virtual void set_bottom(HeapWord* new_bottom) {
+ assert(new_bottom <= _end, "new_bottom > _end");
+ _bottom = new_bottom;
+ resize(pointer_delta(_end, _bottom));
+ }
+
+ // Requires "addr" to be contained by a block, and returns the address of
+ // the start of that block. (May have side effects, namely updating of
+ // shared array entries that "point" too far backwards. This can occur,
+ // for example, when LAB allocation is used in a space covered by the
+ // table.)
+ virtual HeapWord* block_start_unsafe(const void* addr) = 0;
+ // Same as above, but does not have any of the possible side effects
+ // discussed above.
+ virtual HeapWord* block_start_unsafe_const(const void* addr) const = 0;
+
+ // Returns the address of the start of the block containing "addr", or
+ // else "null" if it is covered by no block. (May have side effects,
+ // namely updating of shared array entries that "point" too far
+ // backwards. This can occur, for example, when lab allocation is used
+ // in a space covered by the table.)
+ inline HeapWord* block_start(const void* addr);
+ // Same as above, but does not have any of the possible side effects
+ // discussed above.
+ inline HeapWord* block_start_const(const void* addr) const;
+};
+
+// This implementation of "G1BlockOffsetTable" divides the covered region
+// into "N"-word subregions (where "N" = 2^"LogN". An array with an entry
+// for each such subregion indicates how far back one must go to find the
+// start of the chunk that includes the first word of the subregion.
+//
+// Each BlockOffsetArray is owned by a Space. However, the actual array
+// may be shared by several BlockOffsetArrays; this is useful
+// when a single resizable area (such as a generation) is divided up into
+// several spaces in which contiguous allocation takes place,
+// such as, for example, in G1 or in the train generation.)
+
+// Here is the shared array type.
+
+class G1BlockOffsetSharedArray: public CHeapObj {
+ friend class G1BlockOffsetArray;
+ friend class G1BlockOffsetArrayContigSpace;
+ friend class VMStructs;
+
+private:
+ // The reserved region covered by the shared array.
+ MemRegion _reserved;
+
+ // End of the current committed region.
+ HeapWord* _end;
+
+ // Array for keeping offsets for retrieving object start fast given an
+ // address.
+ VirtualSpace _vs;
+ u_char* _offset_array; // byte array keeping backwards offsets
+
+ // Bounds checking accessors:
+ // For performance these have to devolve to array accesses in product builds.
+ u_char offset_array(size_t index) const {
+ assert(index < _vs.committed_size(), "index out of range");
+ return _offset_array[index];
+ }
+
+ void set_offset_array(size_t index, u_char offset) {
+ assert(index < _vs.committed_size(), "index out of range");
+ assert(offset <= N_words, "offset too large");
+ _offset_array[index] = offset;
+ }
+
+ void set_offset_array(size_t index, HeapWord* high, HeapWord* low) {
+ assert(index < _vs.committed_size(), "index out of range");
+ assert(high >= low, "addresses out of order");
+ assert(pointer_delta(high, low) <= N_words, "offset too large");
+ _offset_array[index] = (u_char) pointer_delta(high, low);
+ }
+
+ void set_offset_array(HeapWord* left, HeapWord* right, u_char offset) {
+ assert(index_for(right - 1) < _vs.committed_size(),
+ "right address out of range");
+ assert(left < right, "Heap addresses out of order");
+ size_t num_cards = pointer_delta(right, left) >> LogN_words;
+ memset(&_offset_array[index_for(left)], offset, num_cards);
+ }
+
+ void set_offset_array(size_t left, size_t right, u_char offset) {
+ assert(right < _vs.committed_size(), "right address out of range");
+ assert(left <= right, "indexes out of order");
+ size_t num_cards = right - left + 1;
+ memset(&_offset_array[left], offset, num_cards);
+ }
+
+ void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const {
+ assert(index < _vs.committed_size(), "index out of range");
+ assert(high >= low, "addresses out of order");
+ assert(pointer_delta(high, low) <= N_words, "offset too large");
+ assert(_offset_array[index] == pointer_delta(high, low),
+ "Wrong offset");
+ }
+
+ bool is_card_boundary(HeapWord* p) const;
+
+ // Return the number of slots needed for an offset array
+ // that covers mem_region_words words.
+ // We always add an extra slot because if an object
+ // ends on a card boundary we put a 0 in the next
+ // offset array slot, so we want that slot always
+ // to be reserved.
+
+ size_t compute_size(size_t mem_region_words) {
+ size_t number_of_slots = (mem_region_words / N_words) + 1;
+ return ReservedSpace::page_align_size_up(number_of_slots);
+ }
+
+public:
+ enum SomePublicConstants {
+ LogN = 9,
+ LogN_words = LogN - LogHeapWordSize,
+ N_bytes = 1 << LogN,
+ N_words = 1 << LogN_words
+ };
+
+ // Initialize the table to cover from "base" to (at least)
+ // "base + init_word_size". In the future, the table may be expanded
+ // (see "resize" below) up to the size of "_reserved" (which must be at
+ // least "init_word_size".) The contents of the initial table are
+ // undefined; it is the responsibility of the constituent
+ // G1BlockOffsetTable(s) to initialize cards.
+ G1BlockOffsetSharedArray(MemRegion reserved, size_t init_word_size);
+
+ // Notes a change in the committed size of the region covered by the
+ // table. The "new_word_size" may not be larger than the size of the
+ // reserved region this table covers.
+ void resize(size_t new_word_size);
+
+ void set_bottom(HeapWord* new_bottom);
+
+ // Updates all the BlockOffsetArray's sharing this shared array to
+ // reflect the current "top"'s of their spaces.
+ void update_offset_arrays();
+
+ // Return the appropriate index into "_offset_array" for "p".
+ inline size_t index_for(const void* p) const;
+
+ // Return the address indicating the start of the region corresponding to
+ // "index" in "_offset_array".
+ inline HeapWord* address_for_index(size_t index) const;
+};
+
+// And here is the G1BlockOffsetTable subtype that uses the array.
+
+class G1BlockOffsetArray: public G1BlockOffsetTable {
+ friend class G1BlockOffsetSharedArray;
+ friend class G1BlockOffsetArrayContigSpace;
+ friend class VMStructs;
+private:
+ enum SomePrivateConstants {
+ N_words = G1BlockOffsetSharedArray::N_words,
+ LogN = G1BlockOffsetSharedArray::LogN
+ };
+
+ // The following enums are used by do_block_helper
+ enum Action {
+ Action_single, // BOT records a single block (see single_block())
+ Action_mark, // BOT marks the start of a block (see mark_block())
+ Action_check // Check that BOT records block correctly
+ // (see verify_single_block()).
+ };
+
+ // This is the array, which can be shared by several BlockOffsetArray's
+ // servicing different
+ G1BlockOffsetSharedArray* _array;
+
+ // The space that owns this subregion.
+ Space* _sp;
+
+ // If "_sp" is a contiguous space, the field below is the view of "_sp"
+ // as a contiguous space, else NULL.
+ ContiguousSpace* _csp;
+
+ // If true, array entries are initialized to 0; otherwise, they are
+ // initialized to point backwards to the beginning of the covered region.
+ bool _init_to_zero;
+
+ // The portion [_unallocated_block, _sp.end()) of the space that
+ // is a single block known not to contain any objects.
+ // NOTE: See BlockOffsetArrayUseUnallocatedBlock flag.
+ HeapWord* _unallocated_block;
+
+ // Sets the entries
+ // corresponding to the cards starting at "start" and ending at "end"
+ // to point back to the card before "start": the interval [start, end)
+ // is right-open.
+ void set_remainder_to_point_to_start(HeapWord* start, HeapWord* end);
+ // Same as above, except that the args here are a card _index_ interval
+ // that is closed: [start_index, end_index]
+ void set_remainder_to_point_to_start_incl(size_t start, size_t end);
+
+ // A helper function for BOT adjustment/verification work
+ void do_block_internal(HeapWord* blk_start, HeapWord* blk_end, Action action);
+
+protected:
+
+ ContiguousSpace* csp() const { return _csp; }
+
+ // Returns the address of a block whose start is at most "addr".
+ // If "has_max_index" is true, "assumes "max_index" is the last valid one
+ // in the array.
+ inline HeapWord* block_at_or_preceding(const void* addr,
+ bool has_max_index,
+ size_t max_index) const;
+
+ // "q" is a block boundary that is <= "addr"; "n" is the address of the
+ // next block (or the end of the space.) Return the address of the
+ // beginning of the block that contains "addr". Does so without side
+ // effects (see, e.g., spec of block_start.)
+ inline HeapWord*
+ forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n,
+ const void* addr) const;
+
+ // "q" is a block boundary that is <= "addr"; return the address of the
+ // beginning of the block that contains "addr". May have side effects
+ // on "this", by updating imprecise entries.
+ inline HeapWord* forward_to_block_containing_addr(HeapWord* q,
+ const void* addr);
+
+ // "q" is a block boundary that is <= "addr"; "n" is the address of the
+ // next block (or the end of the space.) Return the address of the
+ // beginning of the block that contains "addr". May have side effects
+ // on "this", by updating imprecise entries.
+ HeapWord* forward_to_block_containing_addr_slow(HeapWord* q,
+ HeapWord* n,
+ const void* addr);
+
+ // Requires that "*threshold_" be the first array entry boundary at or
+ // above "blk_start", and that "*index_" be the corresponding array
+ // index. If the block starts at or crosses "*threshold_", records
+ // "blk_start" as the appropriate block start for the array index
+ // starting at "*threshold_", and for any other indices crossed by the
+ // block. Updates "*threshold_" and "*index_" to correspond to the first
+ // index after the block end.
+ void alloc_block_work2(HeapWord** threshold_, size_t* index_,
+ HeapWord* blk_start, HeapWord* blk_end);
+
+public:
+ // The space may not have it's bottom and top set yet, which is why the
+ // region is passed as a parameter. If "init_to_zero" is true, the
+ // elements of the array are initialized to zero. Otherwise, they are
+ // initialized to point backwards to the beginning.
+ G1BlockOffsetArray(G1BlockOffsetSharedArray* array, MemRegion mr,
+ bool init_to_zero);
+
+ // Note: this ought to be part of the constructor, but that would require
+ // "this" to be passed as a parameter to a member constructor for
+ // the containing concrete subtype of Space.
+ // This would be legal C++, but MS VC++ doesn't allow it.
+ void set_space(Space* sp);
+
+ // Resets the covered region to the given "mr".
+ void set_region(MemRegion mr);
+
+ // Resets the covered region to one with the same _bottom as before but
+ // the "new_word_size".
+ void resize(size_t new_word_size);
+
+ // These must be guaranteed to work properly (i.e., do nothing)
+ // when "blk_start" ("blk" for second version) is "NULL".
+ virtual void alloc_block(HeapWord* blk_start, HeapWord* blk_end);
+ virtual void alloc_block(HeapWord* blk, size_t size) {
+ alloc_block(blk, blk + size);
+ }
+
+ // The following methods are useful and optimized for a
+ // general, non-contiguous space.
+
+ // The given arguments are required to be the starts of adjacent ("blk1"
+ // before "blk2") well-formed blocks covered by "this". After this call,
+ // they should be considered to form one block.
+ virtual void join_blocks(HeapWord* blk1, HeapWord* blk2);
+
+ // Given a block [blk_start, blk_start + full_blk_size), and
+ // a left_blk_size < full_blk_size, adjust the BOT to show two
+ // blocks [blk_start, blk_start + left_blk_size) and
+ // [blk_start + left_blk_size, blk_start + full_blk_size).
+ // It is assumed (and verified in the non-product VM) that the
+ // BOT was correct for the original block.
+ void split_block(HeapWord* blk_start, size_t full_blk_size,
+ size_t left_blk_size);
+
+ // Adjust the BOT to show that it has a single block in the
+ // range [blk_start, blk_start + size). All necessary BOT
+ // cards are adjusted, but _unallocated_block isn't.
+ void single_block(HeapWord* blk_start, HeapWord* blk_end);
+ void single_block(HeapWord* blk, size_t size) {
+ single_block(blk, blk + size);
+ }
+
+ // Adjust BOT to show that it has a block in the range
+ // [blk_start, blk_start + size). Only the first card
+ // of BOT is touched. It is assumed (and verified in the
+ // non-product VM) that the remaining cards of the block
+ // are correct.
+ void mark_block(HeapWord* blk_start, HeapWord* blk_end);
+ void mark_block(HeapWord* blk, size_t size) {
+ mark_block(blk, blk + size);
+ }
+
+ // Adjust _unallocated_block to indicate that a particular
+ // block has been newly allocated or freed. It is assumed (and
+ // verified in the non-product VM) that the BOT is correct for
+ // the given block.
+ inline void allocated(HeapWord* blk_start, HeapWord* blk_end) {
+ // Verify that the BOT shows [blk, blk + blk_size) to be one block.
+ verify_single_block(blk_start, blk_end);
+ if (BlockOffsetArrayUseUnallocatedBlock) {
+ _unallocated_block = MAX2(_unallocated_block, blk_end);
+ }
+ }
+
+ inline void allocated(HeapWord* blk, size_t size) {
+ allocated(blk, blk + size);
+ }
+
+ inline void freed(HeapWord* blk_start, HeapWord* blk_end);
+
+ inline void freed(HeapWord* blk, size_t size);
+
+ virtual HeapWord* block_start_unsafe(const void* addr);
+ virtual HeapWord* block_start_unsafe_const(const void* addr) const;
+
+ // Requires "addr" to be the start of a card and returns the
+ // start of the block that contains the given address.
+ HeapWord* block_start_careful(const void* addr) const;
+
+ // If true, initialize array slots with no allocated blocks to zero.
+ // Otherwise, make them point back to the front.
+ bool init_to_zero() { return _init_to_zero; }
+
+ // Verification & debugging - ensure that the offset table reflects the fact
+ // that the block [blk_start, blk_end) or [blk, blk + size) is a
+ // single block of storage. NOTE: can;t const this because of
+ // call to non-const do_block_internal() below.
+ inline void verify_single_block(HeapWord* blk_start, HeapWord* blk_end) {
+ if (VerifyBlockOffsetArray) {
+ do_block_internal(blk_start, blk_end, Action_check);
+ }
+ }
+
+ inline void verify_single_block(HeapWord* blk, size_t size) {
+ verify_single_block(blk, blk + size);
+ }
+
+ // Verify that the given block is before _unallocated_block
+ inline void verify_not_unallocated(HeapWord* blk_start,
+ HeapWord* blk_end) const {
+ if (BlockOffsetArrayUseUnallocatedBlock) {
+ assert(blk_start < blk_end, "Block inconsistency?");
+ assert(blk_end <= _unallocated_block, "_unallocated_block problem");
+ }
+ }
+
+ inline void verify_not_unallocated(HeapWord* blk, size_t size) const {
+ verify_not_unallocated(blk, blk + size);
+ }
+
+ void check_all_cards(size_t left_card, size_t right_card) const;
+};
+
+// A subtype of BlockOffsetArray that takes advantage of the fact
+// that its underlying space is a ContiguousSpace, so that its "active"
+// region can be more efficiently tracked (than for a non-contiguous space).
+class G1BlockOffsetArrayContigSpace: public G1BlockOffsetArray {
+ friend class VMStructs;
+
+ // allocation boundary at which offset array must be updated
+ HeapWord* _next_offset_threshold;
+ size_t _next_offset_index; // index corresponding to that boundary
+
+ // Work function to be called when allocation start crosses the next
+ // threshold in the contig space.
+ void alloc_block_work1(HeapWord* blk_start, HeapWord* blk_end) {
+ alloc_block_work2(&_next_offset_threshold, &_next_offset_index,
+ blk_start, blk_end);
+ }
+
+
+ public:
+ G1BlockOffsetArrayContigSpace(G1BlockOffsetSharedArray* array, MemRegion mr);
+
+ // Initialize the threshold to reflect the first boundary after the
+ // bottom of the covered region.
+ HeapWord* initialize_threshold();
+
+ // Zero out the entry for _bottom (offset will be zero).
+ void zero_bottom_entry();
+
+ // Return the next threshold, the point at which the table should be
+ // updated.
+ HeapWord* threshold() const { return _next_offset_threshold; }
+
+ // These must be guaranteed to work properly (i.e., do nothing)
+ // when "blk_start" ("blk" for second version) is "NULL". In this
+ // implementation, that's true because NULL is represented as 0, and thus
+ // never exceeds the "_next_offset_threshold".
+ void alloc_block(HeapWord* blk_start, HeapWord* blk_end) {
+ if (blk_end > _next_offset_threshold)
+ alloc_block_work1(blk_start, blk_end);
+ }
+ void alloc_block(HeapWord* blk, size_t size) {
+ alloc_block(blk, blk+size);
+ }
+
+ HeapWord* block_start_unsafe(const void* addr);
+ HeapWord* block_start_unsafe_const(const void* addr) const;
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp
new file mode 100644
index 00000000000..45e148532c4
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+inline HeapWord* G1BlockOffsetTable::block_start(const void* addr) {
+ if (addr >= _bottom && addr < _end) {
+ return block_start_unsafe(addr);
+ } else {
+ return NULL;
+ }
+}
+
+inline HeapWord*
+G1BlockOffsetTable::block_start_const(const void* addr) const {
+ if (addr >= _bottom && addr < _end) {
+ return block_start_unsafe_const(addr);
+ } else {
+ return NULL;
+ }
+}
+
+inline size_t G1BlockOffsetSharedArray::index_for(const void* p) const {
+ char* pc = (char*)p;
+ assert(pc >= (char*)_reserved.start() &&
+ pc < (char*)_reserved.end(),
+ "p not in range.");
+ size_t delta = pointer_delta(pc, _reserved.start(), sizeof(char));
+ size_t result = delta >> LogN;
+ assert(result < _vs.committed_size(), "bad index from address");
+ return result;
+}
+
+inline HeapWord*
+G1BlockOffsetSharedArray::address_for_index(size_t index) const {
+ assert(index < _vs.committed_size(), "bad index");
+ HeapWord* result = _reserved.start() + (index << LogN_words);
+ assert(result >= _reserved.start() && result < _reserved.end(),
+ "bad address from index");
+ return result;
+}
+
+inline HeapWord*
+G1BlockOffsetArray::block_at_or_preceding(const void* addr,
+ bool has_max_index,
+ size_t max_index) const {
+ assert(_array->offset_array(0) == 0, "objects can't cross covered areas");
+ size_t index = _array->index_for(addr);
+ // We must make sure that the offset table entry we use is valid. If
+ // "addr" is past the end, start at the last known one and go forward.
+ if (has_max_index) {
+ index = MIN2(index, max_index);
+ }
+ HeapWord* q = _array->address_for_index(index);
+
+ uint offset = _array->offset_array(index); // Extend u_char to uint.
+ while (offset >= N_words) {
+ // The excess of the offset from N_words indicates a power of Base
+ // to go back by.
+ size_t n_cards_back = BlockOffsetArray::entry_to_cards_back(offset);
+ q -= (N_words * n_cards_back);
+ assert(q >= _sp->bottom(), "Went below bottom!");
+ index -= n_cards_back;
+ offset = _array->offset_array(index);
+ }
+ assert(offset < N_words, "offset too large");
+ q -= offset;
+ return q;
+}
+
+inline HeapWord*
+G1BlockOffsetArray::
+forward_to_block_containing_addr_const(HeapWord* q, HeapWord* n,
+ const void* addr) const {
+ if (csp() != NULL) {
+ if (addr >= csp()->top()) return csp()->top();
+ while (n <= addr) {
+ q = n;
+ oop obj = oop(q);
+ if (obj->klass() == NULL) return q;
+ n += obj->size();
+ }
+ } else {
+ while (n <= addr) {
+ q = n;
+ oop obj = oop(q);
+ if (obj->klass() == NULL) return q;
+ n += _sp->block_size(q);
+ }
+ }
+ assert(q <= n, "wrong order for q and addr");
+ assert(addr < n, "wrong order for addr and n");
+ return q;
+}
+
+inline HeapWord*
+G1BlockOffsetArray::forward_to_block_containing_addr(HeapWord* q,
+ const void* addr) {
+ if (oop(q)->klass() == NULL) return q;
+ HeapWord* n = q + _sp->block_size(q);
+ // In the normal case, where the query "addr" is a card boundary, and the
+ // offset table chunks are the same size as cards, the block starting at
+ // "q" will contain addr, so the test below will fail, and we'll fall
+ // through quickly.
+ if (n <= addr) {
+ q = forward_to_block_containing_addr_slow(q, n, addr);
+ }
+ assert(q <= addr, "wrong order for current and arg");
+ return q;
+}
+
+//////////////////////////////////////////////////////////////////////////
+// BlockOffsetArrayNonContigSpace inlines
+//////////////////////////////////////////////////////////////////////////
+inline void G1BlockOffsetArray::freed(HeapWord* blk_start, HeapWord* blk_end) {
+ // Verify that the BOT shows [blk_start, blk_end) to be one block.
+ verify_single_block(blk_start, blk_end);
+ // adjust _unallocated_block upward or downward
+ // as appropriate
+ if (BlockOffsetArrayUseUnallocatedBlock) {
+ assert(_unallocated_block <= _end,
+ "Inconsistent value for _unallocated_block");
+ if (blk_end >= _unallocated_block && blk_start <= _unallocated_block) {
+ // CMS-specific note: a block abutting _unallocated_block to
+ // its left is being freed, a new block is being added or
+ // we are resetting following a compaction
+ _unallocated_block = blk_start;
+ }
+ }
+}
+
+inline void G1BlockOffsetArray::freed(HeapWord* blk, size_t size) {
+ freed(blk, blk + size);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
new file mode 100644
index 00000000000..3aa19ccb85d
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp
@@ -0,0 +1,5497 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_g1CollectedHeap.cpp.incl"
+
+// turn it on so that the contents of the young list (scan-only /
+// to-be-collected) are printed at "strategic" points before / during
+// / after the collection --- this is useful for debugging
+#define SCAN_ONLY_VERBOSE 0
+// CURRENT STATUS
+// This file is under construction. Search for "FIXME".
+
+// INVARIANTS/NOTES
+//
+// All allocation activity covered by the G1CollectedHeap interface is
+// serialized by acquiring the HeapLock. This happens in
+// mem_allocate_work, which all such allocation functions call.
+// (Note that this does not apply to TLAB allocation, which is not part
+// of this interface: it is done by clients of this interface.)
+
+// Local to this file.
+
+// Finds the first HeapRegion.
+// No longer used, but might be handy someday.
+
+class FindFirstRegionClosure: public HeapRegionClosure {
+ HeapRegion* _a_region;
+public:
+ FindFirstRegionClosure() : _a_region(NULL) {}
+ bool doHeapRegion(HeapRegion* r) {
+ _a_region = r;
+ return true;
+ }
+ HeapRegion* result() { return _a_region; }
+};
+
+
+class RefineCardTableEntryClosure: public CardTableEntryClosure {
+ SuspendibleThreadSet* _sts;
+ G1RemSet* _g1rs;
+ ConcurrentG1Refine* _cg1r;
+ bool _concurrent;
+public:
+ RefineCardTableEntryClosure(SuspendibleThreadSet* sts,
+ G1RemSet* g1rs,
+ ConcurrentG1Refine* cg1r) :
+ _sts(sts), _g1rs(g1rs), _cg1r(cg1r), _concurrent(true)
+ {}
+ bool do_card_ptr(jbyte* card_ptr, int worker_i) {
+ _g1rs->concurrentRefineOneCard(card_ptr, worker_i);
+ if (_concurrent && _sts->should_yield()) {
+ // Caller will actually yield.
+ return false;
+ }
+ // Otherwise, we finished successfully; return true.
+ return true;
+ }
+ void set_concurrent(bool b) { _concurrent = b; }
+};
+
+
+class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {
+ int _calls;
+ G1CollectedHeap* _g1h;
+ CardTableModRefBS* _ctbs;
+ int _histo[256];
+public:
+ ClearLoggedCardTableEntryClosure() :
+ _calls(0)
+ {
+ _g1h = G1CollectedHeap::heap();
+ _ctbs = (CardTableModRefBS*)_g1h->barrier_set();
+ for (int i = 0; i < 256; i++) _histo[i] = 0;
+ }
+ bool do_card_ptr(jbyte* card_ptr, int worker_i) {
+ if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
+ _calls++;
+ unsigned char* ujb = (unsigned char*)card_ptr;
+ int ind = (int)(*ujb);
+ _histo[ind]++;
+ *card_ptr = -1;
+ }
+ return true;
+ }
+ int calls() { return _calls; }
+ void print_histo() {
+ gclog_or_tty->print_cr("Card table value histogram:");
+ for (int i = 0; i < 256; i++) {
+ if (_histo[i] != 0) {
+ gclog_or_tty->print_cr(" %d: %d", i, _histo[i]);
+ }
+ }
+ }
+};
+
+class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {
+ int _calls;
+ G1CollectedHeap* _g1h;
+ CardTableModRefBS* _ctbs;
+public:
+ RedirtyLoggedCardTableEntryClosure() :
+ _calls(0)
+ {
+ _g1h = G1CollectedHeap::heap();
+ _ctbs = (CardTableModRefBS*)_g1h->barrier_set();
+ }
+ bool do_card_ptr(jbyte* card_ptr, int worker_i) {
+ if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
+ _calls++;
+ *card_ptr = 0;
+ }
+ return true;
+ }
+ int calls() { return _calls; }
+};
+
+YoungList::YoungList(G1CollectedHeap* g1h)
+ : _g1h(g1h), _head(NULL),
+ _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL),
+ _length(0), _scan_only_length(0),
+ _last_sampled_rs_lengths(0),
+ _survivor_head(NULL), _survivors_tail(NULL), _survivor_length(0)
+{
+ guarantee( check_list_empty(false), "just making sure..." );
+}
+
+void YoungList::push_region(HeapRegion *hr) {
+ assert(!hr->is_young(), "should not already be young");
+ assert(hr->get_next_young_region() == NULL, "cause it should!");
+
+ hr->set_next_young_region(_head);
+ _head = hr;
+
+ hr->set_young();
+ double yg_surv_rate = _g1h->g1_policy()->predict_yg_surv_rate((int)_length);
+ ++_length;
+}
+
+void YoungList::add_survivor_region(HeapRegion* hr) {
+ assert(!hr->is_survivor(), "should not already be for survived");
+ assert(hr->get_next_young_region() == NULL, "cause it should!");
+
+ hr->set_next_young_region(_survivor_head);
+ if (_survivor_head == NULL) {
+ _survivors_tail = hr;
+ }
+ _survivor_head = hr;
+
+ hr->set_survivor();
+ ++_survivor_length;
+}
+
+HeapRegion* YoungList::pop_region() {
+ while (_head != NULL) {
+ assert( length() > 0, "list should not be empty" );
+ HeapRegion* ret = _head;
+ _head = ret->get_next_young_region();
+ ret->set_next_young_region(NULL);
+ --_length;
+ assert(ret->is_young(), "region should be very young");
+
+ // Replace 'Survivor' region type with 'Young'. So the region will
+ // be treated as a young region and will not be 'confused' with
+ // newly created survivor regions.
+ if (ret->is_survivor()) {
+ ret->set_young();
+ }
+
+ if (!ret->is_scan_only()) {
+ return ret;
+ }
+
+ // scan-only, we'll add it to the scan-only list
+ if (_scan_only_tail == NULL) {
+ guarantee( _scan_only_head == NULL, "invariant" );
+
+ _scan_only_head = ret;
+ _curr_scan_only = ret;
+ } else {
+ guarantee( _scan_only_head != NULL, "invariant" );
+ _scan_only_tail->set_next_young_region(ret);
+ }
+ guarantee( ret->get_next_young_region() == NULL, "invariant" );
+ _scan_only_tail = ret;
+
+ // no need to be tagged as scan-only any more
+ ret->set_young();
+
+ ++_scan_only_length;
+ }
+ assert( length() == 0, "list should be empty" );
+ return NULL;
+}
+
+void YoungList::empty_list(HeapRegion* list) {
+ while (list != NULL) {
+ HeapRegion* next = list->get_next_young_region();
+ list->set_next_young_region(NULL);
+ list->uninstall_surv_rate_group();
+ list->set_not_young();
+ list = next;
+ }
+}
+
+void YoungList::empty_list() {
+ assert(check_list_well_formed(), "young list should be well formed");
+
+ empty_list(_head);
+ _head = NULL;
+ _length = 0;
+
+ empty_list(_scan_only_head);
+ _scan_only_head = NULL;
+ _scan_only_tail = NULL;
+ _scan_only_length = 0;
+ _curr_scan_only = NULL;
+
+ empty_list(_survivor_head);
+ _survivor_head = NULL;
+ _survivors_tail = NULL;
+ _survivor_length = 0;
+
+ _last_sampled_rs_lengths = 0;
+
+ assert(check_list_empty(false), "just making sure...");
+}
+
+bool YoungList::check_list_well_formed() {
+ bool ret = true;
+
+ size_t length = 0;
+ HeapRegion* curr = _head;
+ HeapRegion* last = NULL;
+ while (curr != NULL) {
+ if (!curr->is_young() || curr->is_scan_only()) {
+ gclog_or_tty->print_cr("### YOUNG REGION "PTR_FORMAT"-"PTR_FORMAT" "
+ "incorrectly tagged (%d, %d)",
+ curr->bottom(), curr->end(),
+ curr->is_young(), curr->is_scan_only());
+ ret = false;
+ }
+ ++length;
+ last = curr;
+ curr = curr->get_next_young_region();
+ }
+ ret = ret && (length == _length);
+
+ if (!ret) {
+ gclog_or_tty->print_cr("### YOUNG LIST seems not well formed!");
+ gclog_or_tty->print_cr("### list has %d entries, _length is %d",
+ length, _length);
+ }
+
+ bool scan_only_ret = true;
+ length = 0;
+ curr = _scan_only_head;
+ last = NULL;
+ while (curr != NULL) {
+ if (!curr->is_young() || curr->is_scan_only()) {
+ gclog_or_tty->print_cr("### SCAN-ONLY REGION "PTR_FORMAT"-"PTR_FORMAT" "
+ "incorrectly tagged (%d, %d)",
+ curr->bottom(), curr->end(),
+ curr->is_young(), curr->is_scan_only());
+ scan_only_ret = false;
+ }
+ ++length;
+ last = curr;
+ curr = curr->get_next_young_region();
+ }
+ scan_only_ret = scan_only_ret && (length == _scan_only_length);
+
+ if ( (last != _scan_only_tail) ||
+ (_scan_only_head == NULL && _scan_only_tail != NULL) ||
+ (_scan_only_head != NULL && _scan_only_tail == NULL) ) {
+ gclog_or_tty->print_cr("## _scan_only_tail is set incorrectly");
+ scan_only_ret = false;
+ }
+
+ if (_curr_scan_only != NULL && _curr_scan_only != _scan_only_head) {
+ gclog_or_tty->print_cr("### _curr_scan_only is set incorrectly");
+ scan_only_ret = false;
+ }
+
+ if (!scan_only_ret) {
+ gclog_or_tty->print_cr("### SCAN-ONLY LIST seems not well formed!");
+ gclog_or_tty->print_cr("### list has %d entries, _scan_only_length is %d",
+ length, _scan_only_length);
+ }
+
+ return ret && scan_only_ret;
+}
+
+bool YoungList::check_list_empty(bool ignore_scan_only_list,
+ bool check_sample) {
+ bool ret = true;
+
+ if (_length != 0) {
+ gclog_or_tty->print_cr("### YOUNG LIST should have 0 length, not %d",
+ _length);
+ ret = false;
+ }
+ if (check_sample && _last_sampled_rs_lengths != 0) {
+ gclog_or_tty->print_cr("### YOUNG LIST has non-zero last sampled RS lengths");
+ ret = false;
+ }
+ if (_head != NULL) {
+ gclog_or_tty->print_cr("### YOUNG LIST does not have a NULL head");
+ ret = false;
+ }
+ if (!ret) {
+ gclog_or_tty->print_cr("### YOUNG LIST does not seem empty");
+ }
+
+ if (ignore_scan_only_list)
+ return ret;
+
+ bool scan_only_ret = true;
+ if (_scan_only_length != 0) {
+ gclog_or_tty->print_cr("### SCAN-ONLY LIST should have 0 length, not %d",
+ _scan_only_length);
+ scan_only_ret = false;
+ }
+ if (_scan_only_head != NULL) {
+ gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL head");
+ scan_only_ret = false;
+ }
+ if (_scan_only_tail != NULL) {
+ gclog_or_tty->print_cr("### SCAN-ONLY LIST does not have a NULL tail");
+ scan_only_ret = false;
+ }
+ if (!scan_only_ret) {
+ gclog_or_tty->print_cr("### SCAN-ONLY LIST does not seem empty");
+ }
+
+ return ret && scan_only_ret;
+}
+
+void
+YoungList::rs_length_sampling_init() {
+ _sampled_rs_lengths = 0;
+ _curr = _head;
+}
+
+bool
+YoungList::rs_length_sampling_more() {
+ return _curr != NULL;
+}
+
+void
+YoungList::rs_length_sampling_next() {
+ assert( _curr != NULL, "invariant" );
+ _sampled_rs_lengths += _curr->rem_set()->occupied();
+ _curr = _curr->get_next_young_region();
+ if (_curr == NULL) {
+ _last_sampled_rs_lengths = _sampled_rs_lengths;
+ // gclog_or_tty->print_cr("last sampled RS lengths = %d", _last_sampled_rs_lengths);
+ }
+}
+
+void
+YoungList::reset_auxilary_lists() {
+ // We could have just "moved" the scan-only list to the young list.
+ // However, the scan-only list is ordered according to the region
+ // age in descending order, so, by moving one entry at a time, we
+ // ensure that it is recreated in ascending order.
+
+ guarantee( is_empty(), "young list should be empty" );
+ assert(check_list_well_formed(), "young list should be well formed");
+
+ // Add survivor regions to SurvRateGroup.
+ _g1h->g1_policy()->note_start_adding_survivor_regions();
+ for (HeapRegion* curr = _survivor_head;
+ curr != NULL;
+ curr = curr->get_next_young_region()) {
+ _g1h->g1_policy()->set_region_survivors(curr);
+ }
+ _g1h->g1_policy()->note_stop_adding_survivor_regions();
+
+ if (_survivor_head != NULL) {
+ _head = _survivor_head;
+ _length = _survivor_length + _scan_only_length;
+ _survivors_tail->set_next_young_region(_scan_only_head);
+ } else {
+ _head = _scan_only_head;
+ _length = _scan_only_length;
+ }
+
+ for (HeapRegion* curr = _scan_only_head;
+ curr != NULL;
+ curr = curr->get_next_young_region()) {
+ curr->recalculate_age_in_surv_rate_group();
+ }
+ _scan_only_head = NULL;
+ _scan_only_tail = NULL;
+ _scan_only_length = 0;
+ _curr_scan_only = NULL;
+
+ _survivor_head = NULL;
+ _survivors_tail = NULL;
+ _survivor_length = 0;
+ _g1h->g1_policy()->finished_recalculating_age_indexes();
+
+ assert(check_list_well_formed(), "young list should be well formed");
+}
+
+void YoungList::print() {
+ HeapRegion* lists[] = {_head, _scan_only_head, _survivor_head};
+ const char* names[] = {"YOUNG", "SCAN-ONLY", "SURVIVOR"};
+
+ for (unsigned int list = 0; list < ARRAY_SIZE(lists); ++list) {
+ gclog_or_tty->print_cr("%s LIST CONTENTS", names[list]);
+ HeapRegion *curr = lists[list];
+ if (curr == NULL)
+ gclog_or_tty->print_cr(" empty");
+ while (curr != NULL) {
+ gclog_or_tty->print_cr(" [%08x-%08x], t: %08x, P: %08x, N: %08x, C: %08x, "
+ "age: %4d, y: %d, s-o: %d, surv: %d",
+ curr->bottom(), curr->end(),
+ curr->top(),
+ curr->prev_top_at_mark_start(),
+ curr->next_top_at_mark_start(),
+ curr->top_at_conc_mark_count(),
+ curr->age_in_surv_rate_group_cond(),
+ curr->is_young(),
+ curr->is_scan_only(),
+ curr->is_survivor());
+ curr = curr->get_next_young_region();
+ }
+ }
+
+ gclog_or_tty->print_cr("");
+}
+
+void G1CollectedHeap::stop_conc_gc_threads() {
+ _cg1r->cg1rThread()->stop();
+ _czft->stop();
+ _cmThread->stop();
+}
+
+
+void G1CollectedHeap::check_ct_logs_at_safepoint() {
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ CardTableModRefBS* ct_bs = (CardTableModRefBS*)barrier_set();
+
+ // Count the dirty cards at the start.
+ CountNonCleanMemRegionClosure count1(this);
+ ct_bs->mod_card_iterate(&count1);
+ int orig_count = count1.n();
+
+ // First clear the logged cards.
+ ClearLoggedCardTableEntryClosure clear;
+ dcqs.set_closure(&clear);
+ dcqs.apply_closure_to_all_completed_buffers();
+ dcqs.iterate_closure_all_threads(false);
+ clear.print_histo();
+
+ // Now ensure that there's no dirty cards.
+ CountNonCleanMemRegionClosure count2(this);
+ ct_bs->mod_card_iterate(&count2);
+ if (count2.n() != 0) {
+ gclog_or_tty->print_cr("Card table has %d entries; %d originally",
+ count2.n(), orig_count);
+ }
+ guarantee(count2.n() == 0, "Card table should be clean.");
+
+ RedirtyLoggedCardTableEntryClosure redirty;
+ JavaThread::dirty_card_queue_set().set_closure(&redirty);
+ dcqs.apply_closure_to_all_completed_buffers();
+ dcqs.iterate_closure_all_threads(false);
+ gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",
+ clear.calls(), orig_count);
+ guarantee(redirty.calls() == clear.calls(),
+ "Or else mechanism is broken.");
+
+ CountNonCleanMemRegionClosure count3(this);
+ ct_bs->mod_card_iterate(&count3);
+ if (count3.n() != orig_count) {
+ gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",
+ orig_count, count3.n());
+ guarantee(count3.n() >= orig_count, "Should have restored them all.");
+ }
+
+ JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);
+}
+
+// Private class members.
+
+G1CollectedHeap* G1CollectedHeap::_g1h;
+
+// Private methods.
+
+// Finds a HeapRegion that can be used to allocate a given size of block.
+
+
+HeapRegion* G1CollectedHeap::newAllocRegion_work(size_t word_size,
+ bool do_expand,
+ bool zero_filled) {
+ ConcurrentZFThread::note_region_alloc();
+ HeapRegion* res = alloc_free_region_from_lists(zero_filled);
+ if (res == NULL && do_expand) {
+ expand(word_size * HeapWordSize);
+ res = alloc_free_region_from_lists(zero_filled);
+ assert(res == NULL ||
+ (!res->isHumongous() &&
+ (!zero_filled ||
+ res->zero_fill_state() == HeapRegion::Allocated)),
+ "Alloc Regions must be zero filled (and non-H)");
+ }
+ if (res != NULL && res->is_empty()) _free_regions--;
+ assert(res == NULL ||
+ (!res->isHumongous() &&
+ (!zero_filled ||
+ res->zero_fill_state() == HeapRegion::Allocated)),
+ "Non-young alloc Regions must be zero filled (and non-H)");
+
+ if (G1TraceRegions) {
+ if (res != NULL) {
+ gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "
+ "top "PTR_FORMAT,
+ res->hrs_index(), res->bottom(), res->end(), res->top());
+ }
+ }
+
+ return res;
+}
+
+HeapRegion* G1CollectedHeap::newAllocRegionWithExpansion(int purpose,
+ size_t word_size,
+ bool zero_filled) {
+ HeapRegion* alloc_region = NULL;
+ if (_gc_alloc_region_counts[purpose] < g1_policy()->max_regions(purpose)) {
+ alloc_region = newAllocRegion_work(word_size, true, zero_filled);
+ if (purpose == GCAllocForSurvived && alloc_region != NULL) {
+ _young_list->add_survivor_region(alloc_region);
+ }
+ ++_gc_alloc_region_counts[purpose];
+ } else {
+ g1_policy()->note_alloc_region_limit_reached(purpose);
+ }
+ return alloc_region;
+}
+
+// If could fit into free regions w/o expansion, try.
+// Otherwise, if can expand, do so.
+// Otherwise, if using ex regions might help, try with ex given back.
+HeapWord* G1CollectedHeap::humongousObjAllocate(size_t word_size) {
+ assert(regions_accounted_for(), "Region leakage!");
+
+ // We can't allocate H regions while cleanupComplete is running, since
+ // some of the regions we find to be empty might not yet be added to the
+ // unclean list. (If we're already at a safepoint, this call is
+ // unnecessary, not to mention wrong.)
+ if (!SafepointSynchronize::is_at_safepoint())
+ wait_for_cleanup_complete();
+
+ size_t num_regions =
+ round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;
+
+ // Special case if < one region???
+
+ // Remember the ft size.
+ size_t x_size = expansion_regions();
+
+ HeapWord* res = NULL;
+ bool eliminated_allocated_from_lists = false;
+
+ // Can the allocation potentially fit in the free regions?
+ if (free_regions() >= num_regions) {
+ res = _hrs->obj_allocate(word_size);
+ }
+ if (res == NULL) {
+ // Try expansion.
+ size_t fs = _hrs->free_suffix();
+ if (fs + x_size >= num_regions) {
+ expand((num_regions - fs) * HeapRegion::GrainBytes);
+ res = _hrs->obj_allocate(word_size);
+ assert(res != NULL, "This should have worked.");
+ } else {
+ // Expansion won't help. Are there enough free regions if we get rid
+ // of reservations?
+ size_t avail = free_regions();
+ if (avail >= num_regions) {
+ res = _hrs->obj_allocate(word_size);
+ if (res != NULL) {
+ remove_allocated_regions_from_lists();
+ eliminated_allocated_from_lists = true;
+ }
+ }
+ }
+ }
+ if (res != NULL) {
+ // Increment by the number of regions allocated.
+ // FIXME: Assumes regions all of size GrainBytes.
+#ifndef PRODUCT
+ mr_bs()->verify_clean_region(MemRegion(res, res + num_regions *
+ HeapRegion::GrainWords));
+#endif
+ if (!eliminated_allocated_from_lists)
+ remove_allocated_regions_from_lists();
+ _summary_bytes_used += word_size * HeapWordSize;
+ _free_regions -= num_regions;
+ _num_humongous_regions += (int) num_regions;
+ }
+ assert(regions_accounted_for(), "Region Leakage");
+ return res;
+}
+
+HeapWord*
+G1CollectedHeap::attempt_allocation_slow(size_t word_size,
+ bool permit_collection_pause) {
+ HeapWord* res = NULL;
+ HeapRegion* allocated_young_region = NULL;
+
+ assert( SafepointSynchronize::is_at_safepoint() ||
+ Heap_lock->owned_by_self(), "pre condition of the call" );
+
+ if (isHumongous(word_size)) {
+ // Allocation of a humongous object can, in a sense, complete a
+ // partial region, if the previous alloc was also humongous, and
+ // caused the test below to succeed.
+ if (permit_collection_pause)
+ do_collection_pause_if_appropriate(word_size);
+ res = humongousObjAllocate(word_size);
+ assert(_cur_alloc_region == NULL
+ || !_cur_alloc_region->isHumongous(),
+ "Prevent a regression of this bug.");
+
+ } else {
+ // We may have concurrent cleanup working at the time. Wait for it
+ // to complete. In the future we would probably want to make the
+ // concurrent cleanup truly concurrent by decoupling it from the
+ // allocation.
+ if (!SafepointSynchronize::is_at_safepoint())
+ wait_for_cleanup_complete();
+ // If we do a collection pause, this will be reset to a non-NULL
+ // value. If we don't, nulling here ensures that we allocate a new
+ // region below.
+ if (_cur_alloc_region != NULL) {
+ // We're finished with the _cur_alloc_region.
+ _summary_bytes_used += _cur_alloc_region->used();
+ _cur_alloc_region = NULL;
+ }
+ assert(_cur_alloc_region == NULL, "Invariant.");
+ // Completion of a heap region is perhaps a good point at which to do
+ // a collection pause.
+ if (permit_collection_pause)
+ do_collection_pause_if_appropriate(word_size);
+ // Make sure we have an allocation region available.
+ if (_cur_alloc_region == NULL) {
+ if (!SafepointSynchronize::is_at_safepoint())
+ wait_for_cleanup_complete();
+ bool next_is_young = should_set_young_locked();
+ // If the next region is not young, make sure it's zero-filled.
+ _cur_alloc_region = newAllocRegion(word_size, !next_is_young);
+ if (_cur_alloc_region != NULL) {
+ _summary_bytes_used -= _cur_alloc_region->used();
+ if (next_is_young) {
+ set_region_short_lived_locked(_cur_alloc_region);
+ allocated_young_region = _cur_alloc_region;
+ }
+ }
+ }
+ assert(_cur_alloc_region == NULL || !_cur_alloc_region->isHumongous(),
+ "Prevent a regression of this bug.");
+
+ // Now retry the allocation.
+ if (_cur_alloc_region != NULL) {
+ res = _cur_alloc_region->allocate(word_size);
+ }
+ }
+
+ // NOTE: fails frequently in PRT
+ assert(regions_accounted_for(), "Region leakage!");
+
+ if (res != NULL) {
+ if (!SafepointSynchronize::is_at_safepoint()) {
+ assert( permit_collection_pause, "invariant" );
+ assert( Heap_lock->owned_by_self(), "invariant" );
+ Heap_lock->unlock();
+ }
+
+ if (allocated_young_region != NULL) {
+ HeapRegion* hr = allocated_young_region;
+ HeapWord* bottom = hr->bottom();
+ HeapWord* end = hr->end();
+ MemRegion mr(bottom, end);
+ ((CardTableModRefBS*)_g1h->barrier_set())->dirty(mr);
+ }
+ }
+
+ assert( SafepointSynchronize::is_at_safepoint() ||
+ (res == NULL && Heap_lock->owned_by_self()) ||
+ (res != NULL && !Heap_lock->owned_by_self()),
+ "post condition of the call" );
+
+ return res;
+}
+
+HeapWord*
+G1CollectedHeap::mem_allocate(size_t word_size,
+ bool is_noref,
+ bool is_tlab,
+ bool* gc_overhead_limit_was_exceeded) {
+ debug_only(check_for_valid_allocation_state());
+ assert(no_gc_in_progress(), "Allocation during gc not allowed");
+ HeapWord* result = NULL;
+
+ // Loop until the allocation is satisified,
+ // or unsatisfied after GC.
+ for (int try_count = 1; /* return or throw */; try_count += 1) {
+ int gc_count_before;
+ {
+ Heap_lock->lock();
+ result = attempt_allocation(word_size);
+ if (result != NULL) {
+ // attempt_allocation should have unlocked the heap lock
+ assert(is_in(result), "result not in heap");
+ return result;
+ }
+ // Read the gc count while the heap lock is held.
+ gc_count_before = SharedHeap::heap()->total_collections();
+ Heap_lock->unlock();
+ }
+
+ // Create the garbage collection operation...
+ VM_G1CollectForAllocation op(word_size,
+ gc_count_before);
+
+ // ...and get the VM thread to execute it.
+ VMThread::execute(&op);
+ if (op.prologue_succeeded()) {
+ result = op.result();
+ assert(result == NULL || is_in(result), "result not in heap");
+ return result;
+ }
+
+ // Give a warning if we seem to be looping forever.
+ if ((QueuedAllocationWarningCount > 0) &&
+ (try_count % QueuedAllocationWarningCount == 0)) {
+ warning("G1CollectedHeap::mem_allocate_work retries %d times",
+ try_count);
+ }
+ }
+}
+
+void G1CollectedHeap::abandon_cur_alloc_region() {
+ if (_cur_alloc_region != NULL) {
+ // We're finished with the _cur_alloc_region.
+ if (_cur_alloc_region->is_empty()) {
+ _free_regions++;
+ free_region(_cur_alloc_region);
+ } else {
+ _summary_bytes_used += _cur_alloc_region->used();
+ }
+ _cur_alloc_region = NULL;
+ }
+}
+
+class PostMCRemSetClearClosure: public HeapRegionClosure {
+ ModRefBarrierSet* _mr_bs;
+public:
+ PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}
+ bool doHeapRegion(HeapRegion* r) {
+ r->reset_gc_time_stamp();
+ if (r->continuesHumongous())
+ return false;
+ HeapRegionRemSet* hrrs = r->rem_set();
+ if (hrrs != NULL) hrrs->clear();
+ // You might think here that we could clear just the cards
+ // corresponding to the used region. But no: if we leave a dirty card
+ // in a region we might allocate into, then it would prevent that card
+ // from being enqueued, and cause it to be missed.
+ // Re: the performance cost: we shouldn't be doing full GC anyway!
+ _mr_bs->clear(MemRegion(r->bottom(), r->end()));
+ return false;
+ }
+};
+
+
+class PostMCRemSetInvalidateClosure: public HeapRegionClosure {
+ ModRefBarrierSet* _mr_bs;
+public:
+ PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->continuesHumongous()) return false;
+ if (r->used_region().word_size() != 0) {
+ _mr_bs->invalidate(r->used_region(), true /*whole heap*/);
+ }
+ return false;
+ }
+};
+
+void G1CollectedHeap::do_collection(bool full, bool clear_all_soft_refs,
+ size_t word_size) {
+ ResourceMark rm;
+
+ if (full && DisableExplicitGC) {
+ gclog_or_tty->print("\n\n\nDisabling Explicit GC\n\n\n");
+ return;
+ }
+
+ assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+ assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
+
+ if (GC_locker::is_active()) {
+ return; // GC is disabled (e.g. JNI GetXXXCritical operation)
+ }
+
+ {
+ IsGCActiveMark x;
+
+ // Timing
+ gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
+ TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
+ TraceTime t(full ? "Full GC (System.gc())" : "Full GC", PrintGC, true, gclog_or_tty);
+
+ double start = os::elapsedTime();
+ GCOverheadReporter::recordSTWStart(start);
+ g1_policy()->record_full_collection_start();
+
+ gc_prologue(true);
+ increment_total_collections();
+
+ size_t g1h_prev_used = used();
+ assert(used() == recalculate_used(), "Should be equal");
+
+ if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
+ HandleMark hm; // Discard invalid handles created during verification
+ prepare_for_verify();
+ gclog_or_tty->print(" VerifyBeforeGC:");
+ Universe::verify(true);
+ }
+ assert(regions_accounted_for(), "Region leakage!");
+
+ COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+ // We want to discover references, but not process them yet.
+ // This mode is disabled in
+ // instanceRefKlass::process_discovered_references if the
+ // generation does some collection work, or
+ // instanceRefKlass::enqueue_discovered_references if the
+ // generation returns without doing any work.
+ ref_processor()->disable_discovery();
+ ref_processor()->abandon_partial_discovery();
+ ref_processor()->verify_no_references_recorded();
+
+ // Abandon current iterations of concurrent marking and concurrent
+ // refinement, if any are in progress.
+ concurrent_mark()->abort();
+
+ // Make sure we'll choose a new allocation region afterwards.
+ abandon_cur_alloc_region();
+ assert(_cur_alloc_region == NULL, "Invariant.");
+ g1_rem_set()->as_HRInto_G1RemSet()->cleanupHRRS();
+ tear_down_region_lists();
+ set_used_regions_to_need_zero_fill();
+ if (g1_policy()->in_young_gc_mode()) {
+ empty_young_list();
+ g1_policy()->set_full_young_gcs(true);
+ }
+
+ // Temporarily make reference _discovery_ single threaded (non-MT).
+ ReferenceProcessorMTMutator rp_disc_ser(ref_processor(), false);
+
+ // Temporarily make refs discovery atomic
+ ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);
+
+ // Temporarily clear _is_alive_non_header
+ ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);
+
+ ref_processor()->enable_discovery();
+ ref_processor()->setup_policy(clear_all_soft_refs);
+
+ // Do collection work
+ {
+ HandleMark hm; // Discard invalid handles created during gc
+ G1MarkSweep::invoke_at_safepoint(ref_processor(), clear_all_soft_refs);
+ }
+ // Because freeing humongous regions may have added some unclean
+ // regions, it is necessary to tear down again before rebuilding.
+ tear_down_region_lists();
+ rebuild_region_lists();
+
+ _summary_bytes_used = recalculate_used();
+
+ ref_processor()->enqueue_discovered_references();
+
+ COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+
+ if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
+ HandleMark hm; // Discard invalid handles created during verification
+ gclog_or_tty->print(" VerifyAfterGC:");
+ Universe::verify(false);
+ }
+ NOT_PRODUCT(ref_processor()->verify_no_references_recorded());
+
+ reset_gc_time_stamp();
+ // Since everything potentially moved, we will clear all remembered
+ // sets, and clear all cards. Later we will also cards in the used
+ // portion of the heap after the resizing (which could be a shrinking.)
+ // We will also reset the GC time stamps of the regions.
+ PostMCRemSetClearClosure rs_clear(mr_bs());
+ heap_region_iterate(&rs_clear);
+
+ // Resize the heap if necessary.
+ resize_if_necessary_after_full_collection(full ? 0 : word_size);
+
+ // Since everything potentially moved, we will clear all remembered
+ // sets, but also dirty all cards corresponding to used regions.
+ PostMCRemSetInvalidateClosure rs_invalidate(mr_bs());
+ heap_region_iterate(&rs_invalidate);
+ if (_cg1r->use_cache()) {
+ _cg1r->clear_and_record_card_counts();
+ _cg1r->clear_hot_cache();
+ }
+
+ if (PrintGC) {
+ print_size_transition(gclog_or_tty, g1h_prev_used, used(), capacity());
+ }
+
+ if (true) { // FIXME
+ // Ask the permanent generation to adjust size for full collections
+ perm()->compute_new_size();
+ }
+
+ double end = os::elapsedTime();
+ GCOverheadReporter::recordSTWEnd(end);
+ g1_policy()->record_full_collection_end();
+
+ gc_epilogue(true);
+
+ // Abandon concurrent refinement. This must happen last: in the
+ // dirty-card logging system, some cards may be dirty by weak-ref
+ // processing, and may be enqueued. But the whole card table is
+ // dirtied, so this should abandon those logs, and set "do_traversal"
+ // to true.
+ concurrent_g1_refine()->set_pya_restart();
+
+ assert(regions_accounted_for(), "Region leakage!");
+ }
+
+ if (g1_policy()->in_young_gc_mode()) {
+ _young_list->reset_sampled_info();
+ assert( check_young_list_empty(false, false),
+ "young list should be empty at this point");
+ }
+}
+
+void G1CollectedHeap::do_full_collection(bool clear_all_soft_refs) {
+ do_collection(true, clear_all_soft_refs, 0);
+}
+
+// This code is mostly copied from TenuredGeneration.
+void
+G1CollectedHeap::
+resize_if_necessary_after_full_collection(size_t word_size) {
+ assert(MinHeapFreeRatio <= MaxHeapFreeRatio, "sanity check");
+
+ // Include the current allocation, if any, and bytes that will be
+ // pre-allocated to support collections, as "used".
+ const size_t used_after_gc = used();
+ const size_t capacity_after_gc = capacity();
+ const size_t free_after_gc = capacity_after_gc - used_after_gc;
+
+ // We don't have floating point command-line arguments
+ const double minimum_free_percentage = (double) MinHeapFreeRatio / 100;
+ const double maximum_used_percentage = 1.0 - minimum_free_percentage;
+ const double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;
+ const double minimum_used_percentage = 1.0 - maximum_free_percentage;
+
+ size_t minimum_desired_capacity = (size_t) (used_after_gc / maximum_used_percentage);
+ size_t maximum_desired_capacity = (size_t) (used_after_gc / minimum_used_percentage);
+
+ // Don't shrink less than the initial size.
+ minimum_desired_capacity =
+ MAX2(minimum_desired_capacity,
+ collector_policy()->initial_heap_byte_size());
+ maximum_desired_capacity =
+ MAX2(maximum_desired_capacity,
+ collector_policy()->initial_heap_byte_size());
+
+ // We are failing here because minimum_desired_capacity is
+ assert(used_after_gc <= minimum_desired_capacity, "sanity check");
+ assert(minimum_desired_capacity <= maximum_desired_capacity, "sanity check");
+
+ if (PrintGC && Verbose) {
+ const double free_percentage = ((double)free_after_gc) / capacity();
+ gclog_or_tty->print_cr("Computing new size after full GC ");
+ gclog_or_tty->print_cr(" "
+ " minimum_free_percentage: %6.2f",
+ minimum_free_percentage);
+ gclog_or_tty->print_cr(" "
+ " maximum_free_percentage: %6.2f",
+ maximum_free_percentage);
+ gclog_or_tty->print_cr(" "
+ " capacity: %6.1fK"
+ " minimum_desired_capacity: %6.1fK"
+ " maximum_desired_capacity: %6.1fK",
+ capacity() / (double) K,
+ minimum_desired_capacity / (double) K,
+ maximum_desired_capacity / (double) K);
+ gclog_or_tty->print_cr(" "
+ " free_after_gc : %6.1fK"
+ " used_after_gc : %6.1fK",
+ free_after_gc / (double) K,
+ used_after_gc / (double) K);
+ gclog_or_tty->print_cr(" "
+ " free_percentage: %6.2f",
+ free_percentage);
+ }
+ if (capacity() < minimum_desired_capacity) {
+ // Don't expand unless it's significant
+ size_t expand_bytes = minimum_desired_capacity - capacity_after_gc;
+ expand(expand_bytes);
+ if (PrintGC && Verbose) {
+ gclog_or_tty->print_cr(" expanding:"
+ " minimum_desired_capacity: %6.1fK"
+ " expand_bytes: %6.1fK",
+ minimum_desired_capacity / (double) K,
+ expand_bytes / (double) K);
+ }
+
+ // No expansion, now see if we want to shrink
+ } else if (capacity() > maximum_desired_capacity) {
+ // Capacity too large, compute shrinking size
+ size_t shrink_bytes = capacity_after_gc - maximum_desired_capacity;
+ shrink(shrink_bytes);
+ if (PrintGC && Verbose) {
+ gclog_or_tty->print_cr(" "
+ " shrinking:"
+ " initSize: %.1fK"
+ " maximum_desired_capacity: %.1fK",
+ collector_policy()->initial_heap_byte_size() / (double) K,
+ maximum_desired_capacity / (double) K);
+ gclog_or_tty->print_cr(" "
+ " shrink_bytes: %.1fK",
+ shrink_bytes / (double) K);
+ }
+ }
+}
+
+
+HeapWord*
+G1CollectedHeap::satisfy_failed_allocation(size_t word_size) {
+ HeapWord* result = NULL;
+
+ // In a G1 heap, we're supposed to keep allocation from failing by
+ // incremental pauses. Therefore, at least for now, we'll favor
+ // expansion over collection. (This might change in the future if we can
+ // do something smarter than full collection to satisfy a failed alloc.)
+
+ result = expand_and_allocate(word_size);
+ if (result != NULL) {
+ assert(is_in(result), "result not in heap");
+ return result;
+ }
+
+ // OK, I guess we have to try collection.
+
+ do_collection(false, false, word_size);
+
+ result = attempt_allocation(word_size, /*permit_collection_pause*/false);
+
+ if (result != NULL) {
+ assert(is_in(result), "result not in heap");
+ return result;
+ }
+
+ // Try collecting soft references.
+ do_collection(false, true, word_size);
+ result = attempt_allocation(word_size, /*permit_collection_pause*/false);
+ if (result != NULL) {
+ assert(is_in(result), "result not in heap");
+ return result;
+ }
+
+ // What else? We might try synchronous finalization later. If the total
+ // space available is large enough for the allocation, then a more
+ // complete compaction phase than we've tried so far might be
+ // appropriate.
+ return NULL;
+}
+
+// Attempting to expand the heap sufficiently
+// to support an allocation of the given "word_size". If
+// successful, perform the allocation and return the address of the
+// allocated block, or else "NULL".
+
+HeapWord* G1CollectedHeap::expand_and_allocate(size_t word_size) {
+ size_t expand_bytes = word_size * HeapWordSize;
+ if (expand_bytes < MinHeapDeltaBytes) {
+ expand_bytes = MinHeapDeltaBytes;
+ }
+ expand(expand_bytes);
+ assert(regions_accounted_for(), "Region leakage!");
+ HeapWord* result = attempt_allocation(word_size, false /* permit_collection_pause */);
+ return result;
+}
+
+size_t G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr) {
+ size_t pre_used = 0;
+ size_t cleared_h_regions = 0;
+ size_t freed_regions = 0;
+ UncleanRegionList local_list;
+ free_region_if_totally_empty_work(hr, pre_used, cleared_h_regions,
+ freed_regions, &local_list);
+
+ finish_free_region_work(pre_used, cleared_h_regions, freed_regions,
+ &local_list);
+ return pre_used;
+}
+
+void
+G1CollectedHeap::free_region_if_totally_empty_work(HeapRegion* hr,
+ size_t& pre_used,
+ size_t& cleared_h,
+ size_t& freed_regions,
+ UncleanRegionList* list,
+ bool par) {
+ assert(!hr->continuesHumongous(), "should have filtered these out");
+ size_t res = 0;
+ if (!hr->popular() && hr->used() > 0 && hr->garbage_bytes() == hr->used()) {
+ if (!hr->is_young()) {
+ if (G1PolicyVerbose > 0)
+ gclog_or_tty->print_cr("Freeing empty region "PTR_FORMAT "(" SIZE_FORMAT " bytes)"
+ " during cleanup", hr, hr->used());
+ free_region_work(hr, pre_used, cleared_h, freed_regions, list, par);
+ }
+ }
+}
+
+// FIXME: both this and shrink could probably be more efficient by
+// doing one "VirtualSpace::expand_by" call rather than several.
+void G1CollectedHeap::expand(size_t expand_bytes) {
+ size_t old_mem_size = _g1_storage.committed_size();
+ // We expand by a minimum of 1K.
+ expand_bytes = MAX2(expand_bytes, (size_t)K);
+ size_t aligned_expand_bytes =
+ ReservedSpace::page_align_size_up(expand_bytes);
+ aligned_expand_bytes = align_size_up(aligned_expand_bytes,
+ HeapRegion::GrainBytes);
+ expand_bytes = aligned_expand_bytes;
+ while (expand_bytes > 0) {
+ HeapWord* base = (HeapWord*)_g1_storage.high();
+ // Commit more storage.
+ bool successful = _g1_storage.expand_by(HeapRegion::GrainBytes);
+ if (!successful) {
+ expand_bytes = 0;
+ } else {
+ expand_bytes -= HeapRegion::GrainBytes;
+ // Expand the committed region.
+ HeapWord* high = (HeapWord*) _g1_storage.high();
+ _g1_committed.set_end(high);
+ // Create a new HeapRegion.
+ MemRegion mr(base, high);
+ bool is_zeroed = !_g1_max_committed.contains(base);
+ HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);
+
+ // Now update max_committed if necessary.
+ _g1_max_committed.set_end(MAX2(_g1_max_committed.end(), high));
+
+ // Add it to the HeapRegionSeq.
+ _hrs->insert(hr);
+ // Set the zero-fill state, according to whether it's already
+ // zeroed.
+ {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ if (is_zeroed) {
+ hr->set_zero_fill_complete();
+ put_free_region_on_list_locked(hr);
+ } else {
+ hr->set_zero_fill_needed();
+ put_region_on_unclean_list_locked(hr);
+ }
+ }
+ _free_regions++;
+ // And we used up an expansion region to create it.
+ _expansion_regions--;
+ // Tell the cardtable about it.
+ Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);
+ // And the offset table as well.
+ _bot_shared->resize(_g1_committed.word_size());
+ }
+ }
+ if (Verbose && PrintGC) {
+ size_t new_mem_size = _g1_storage.committed_size();
+ gclog_or_tty->print_cr("Expanding garbage-first heap from %ldK by %ldK to %ldK",
+ old_mem_size/K, aligned_expand_bytes/K,
+ new_mem_size/K);
+ }
+}
+
+void G1CollectedHeap::shrink_helper(size_t shrink_bytes)
+{
+ size_t old_mem_size = _g1_storage.committed_size();
+ size_t aligned_shrink_bytes =
+ ReservedSpace::page_align_size_down(shrink_bytes);
+ aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,
+ HeapRegion::GrainBytes);
+ size_t num_regions_deleted = 0;
+ MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);
+
+ assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");
+ if (mr.byte_size() > 0)
+ _g1_storage.shrink_by(mr.byte_size());
+ assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");
+
+ _g1_committed.set_end(mr.start());
+ _free_regions -= num_regions_deleted;
+ _expansion_regions += num_regions_deleted;
+
+ // Tell the cardtable about it.
+ Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);
+
+ // And the offset table as well.
+ _bot_shared->resize(_g1_committed.word_size());
+
+ HeapRegionRemSet::shrink_heap(n_regions());
+
+ if (Verbose && PrintGC) {
+ size_t new_mem_size = _g1_storage.committed_size();
+ gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",
+ old_mem_size/K, aligned_shrink_bytes/K,
+ new_mem_size/K);
+ }
+}
+
+void G1CollectedHeap::shrink(size_t shrink_bytes) {
+ release_gc_alloc_regions();
+ tear_down_region_lists(); // We will rebuild them in a moment.
+ shrink_helper(shrink_bytes);
+ rebuild_region_lists();
+}
+
+// Public methods.
+
+#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
+#pragma warning( disable:4355 ) // 'this' : used in base member initializer list
+#endif // _MSC_VER
+
+
+G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :
+ SharedHeap(policy_),
+ _g1_policy(policy_),
+ _ref_processor(NULL),
+ _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),
+ _bot_shared(NULL),
+ _par_alloc_during_gc_lock(Mutex::leaf, "par alloc during GC lock"),
+ _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),
+ _evac_failure_scan_stack(NULL) ,
+ _mark_in_progress(false),
+ _cg1r(NULL), _czft(NULL), _summary_bytes_used(0),
+ _cur_alloc_region(NULL),
+ _refine_cte_cl(NULL),
+ _free_region_list(NULL), _free_region_list_size(0),
+ _free_regions(0),
+ _popular_object_boundary(NULL),
+ _cur_pop_hr_index(0),
+ _popular_regions_to_be_evacuated(NULL),
+ _pop_obj_rc_at_copy(),
+ _full_collection(false),
+ _unclean_region_list(),
+ _unclean_regions_coming(false),
+ _young_list(new YoungList(this)),
+ _gc_time_stamp(0),
+ _surviving_young_words(NULL)
+{
+ _g1h = this; // To catch bugs.
+ if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {
+ vm_exit_during_initialization("Failed necessary allocation.");
+ }
+ int n_queues = MAX2((int)ParallelGCThreads, 1);
+ _task_queues = new RefToScanQueueSet(n_queues);
+
+ int n_rem_sets = HeapRegionRemSet::num_par_rem_sets();
+ assert(n_rem_sets > 0, "Invariant.");
+
+ HeapRegionRemSetIterator** iter_arr =
+ NEW_C_HEAP_ARRAY(HeapRegionRemSetIterator*, n_queues);
+ for (int i = 0; i < n_queues; i++) {
+ iter_arr[i] = new HeapRegionRemSetIterator();
+ }
+ _rem_set_iterator = iter_arr;
+
+ for (int i = 0; i < n_queues; i++) {
+ RefToScanQueue* q = new RefToScanQueue();
+ q->initialize();
+ _task_queues->register_queue(i, q);
+ }
+
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ _gc_alloc_regions[ap] = NULL;
+ _gc_alloc_region_counts[ap] = 0;
+ }
+ guarantee(_task_queues != NULL, "task_queues allocation failure.");
+}
+
+jint G1CollectedHeap::initialize() {
+ os::enable_vtime();
+
+ // Necessary to satisfy locking discipline assertions.
+
+ MutexLocker x(Heap_lock);
+
+ // While there are no constraints in the GC code that HeapWordSize
+ // be any particular value, there are multiple other areas in the
+ // system which believe this to be true (e.g. oop->object_size in some
+ // cases incorrectly returns the size in wordSize units rather than
+ // HeapWordSize).
+ guarantee(HeapWordSize == wordSize, "HeapWordSize must equal wordSize");
+
+ size_t init_byte_size = collector_policy()->initial_heap_byte_size();
+ size_t max_byte_size = collector_policy()->max_heap_byte_size();
+
+ // Ensure that the sizes are properly aligned.
+ Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap");
+ Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap");
+
+ // We allocate this in any case, but only do no work if the command line
+ // param is off.
+ _cg1r = new ConcurrentG1Refine();
+
+ // Reserve the maximum.
+ PermanentGenerationSpec* pgs = collector_policy()->permanent_generation();
+ // Includes the perm-gen.
+ ReservedSpace heap_rs(max_byte_size + pgs->max_size(),
+ HeapRegion::GrainBytes,
+ false /*ism*/);
+
+ if (!heap_rs.is_reserved()) {
+ vm_exit_during_initialization("Could not reserve enough space for object heap");
+ return JNI_ENOMEM;
+ }
+
+ // It is important to do this in a way such that concurrent readers can't
+ // temporarily think somethings in the heap. (I've actually seen this
+ // happen in asserts: DLD.)
+ _reserved.set_word_size(0);
+ _reserved.set_start((HeapWord*)heap_rs.base());
+ _reserved.set_end((HeapWord*)(heap_rs.base() + heap_rs.size()));
+
+ _expansion_regions = max_byte_size/HeapRegion::GrainBytes;
+
+ _num_humongous_regions = 0;
+
+ // Create the gen rem set (and barrier set) for the entire reserved region.
+ _rem_set = collector_policy()->create_rem_set(_reserved, 2);
+ set_barrier_set(rem_set()->bs());
+ if (barrier_set()->is_a(BarrierSet::ModRef)) {
+ _mr_bs = (ModRefBarrierSet*)_barrier_set;
+ } else {
+ vm_exit_during_initialization("G1 requires a mod ref bs.");
+ return JNI_ENOMEM;
+ }
+
+ // Also create a G1 rem set.
+ if (G1UseHRIntoRS) {
+ if (mr_bs()->is_a(BarrierSet::CardTableModRef)) {
+ _g1_rem_set = new HRInto_G1RemSet(this, (CardTableModRefBS*)mr_bs());
+ } else {
+ vm_exit_during_initialization("G1 requires a cardtable mod ref bs.");
+ return JNI_ENOMEM;
+ }
+ } else {
+ _g1_rem_set = new StupidG1RemSet(this);
+ }
+
+ // Carve out the G1 part of the heap.
+
+ ReservedSpace g1_rs = heap_rs.first_part(max_byte_size);
+ _g1_reserved = MemRegion((HeapWord*)g1_rs.base(),
+ g1_rs.size()/HeapWordSize);
+ ReservedSpace perm_gen_rs = heap_rs.last_part(max_byte_size);
+
+ _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());
+
+ _g1_storage.initialize(g1_rs, 0);
+ _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);
+ _g1_max_committed = _g1_committed;
+ _hrs = new HeapRegionSeq(_expansion_regions);
+ guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");
+ guarantee(_cur_alloc_region == NULL, "from constructor");
+
+ _bot_shared = new G1BlockOffsetSharedArray(_reserved,
+ heap_word_size(init_byte_size));
+
+ _g1h = this;
+
+ // Create the ConcurrentMark data structure and thread.
+ // (Must do this late, so that "max_regions" is defined.)
+ _cm = new ConcurrentMark(heap_rs, (int) max_regions());
+ _cmThread = _cm->cmThread();
+
+ // ...and the concurrent zero-fill thread, if necessary.
+ if (G1ConcZeroFill) {
+ _czft = new ConcurrentZFThread();
+ }
+
+
+
+ // Allocate the popular regions; take them off free lists.
+ size_t pop_byte_size = G1NumPopularRegions * HeapRegion::GrainBytes;
+ expand(pop_byte_size);
+ _popular_object_boundary =
+ _g1_reserved.start() + (G1NumPopularRegions * HeapRegion::GrainWords);
+ for (int i = 0; i < G1NumPopularRegions; i++) {
+ HeapRegion* hr = newAllocRegion(HeapRegion::GrainWords);
+ // assert(hr != NULL && hr->bottom() < _popular_object_boundary,
+ // "Should be enough, and all should be below boundary.");
+ hr->set_popular(true);
+ }
+ assert(_cur_pop_hr_index == 0, "Start allocating at the first region.");
+
+ // Initialize the from_card cache structure of HeapRegionRemSet.
+ HeapRegionRemSet::init_heap(max_regions());
+
+ // Now expand into the rest of the initial heap size.
+ expand(init_byte_size - pop_byte_size);
+
+ // Perform any initialization actions delegated to the policy.
+ g1_policy()->init();
+
+ g1_policy()->note_start_of_mark_thread();
+
+ _refine_cte_cl =
+ new RefineCardTableEntryClosure(ConcurrentG1RefineThread::sts(),
+ g1_rem_set(),
+ concurrent_g1_refine());
+ JavaThread::dirty_card_queue_set().set_closure(_refine_cte_cl);
+
+ JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
+ SATB_Q_FL_lock,
+ 0,
+ Shared_SATB_Q_lock);
+ if (G1RSBarrierUseQueue) {
+ JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon,
+ DirtyCardQ_FL_lock,
+ G1DirtyCardQueueMax,
+ Shared_DirtyCardQ_lock);
+ }
+ // In case we're keeping closure specialization stats, initialize those
+ // counts and that mechanism.
+ SpecializationStats::clear();
+
+ _gc_alloc_region_list = NULL;
+
+ // Do later initialization work for concurrent refinement.
+ _cg1r->init();
+
+ const char* group_names[] = { "CR", "ZF", "CM", "CL" };
+ GCOverheadReporter::initGCOverheadReporter(4, group_names);
+
+ return JNI_OK;
+}
+
+void G1CollectedHeap::ref_processing_init() {
+ SharedHeap::ref_processing_init();
+ MemRegion mr = reserved_region();
+ _ref_processor = ReferenceProcessor::create_ref_processor(
+ mr, // span
+ false, // Reference discovery is not atomic
+ // (though it shouldn't matter here.)
+ true, // mt_discovery
+ NULL, // is alive closure: need to fill this in for efficiency
+ ParallelGCThreads,
+ ParallelRefProcEnabled,
+ true); // Setting next fields of discovered
+ // lists requires a barrier.
+}
+
+size_t G1CollectedHeap::capacity() const {
+ return _g1_committed.byte_size();
+}
+
+void G1CollectedHeap::iterate_dirty_card_closure(bool concurrent,
+ int worker_i) {
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ int n_completed_buffers = 0;
+ while (dcqs.apply_closure_to_completed_buffer(worker_i, 0, true)) {
+ n_completed_buffers++;
+ }
+ g1_policy()->record_update_rs_processed_buffers(worker_i,
+ (double) n_completed_buffers);
+ dcqs.clear_n_completed_buffers();
+ // Finish up the queue...
+ if (worker_i == 0) concurrent_g1_refine()->clean_up_cache(worker_i,
+ g1_rem_set());
+ assert(!dcqs.completed_buffers_exist_dirty(), "Completed buffers exist!");
+}
+
+
+// Computes the sum of the storage used by the various regions.
+
+size_t G1CollectedHeap::used() const {
+ assert(Heap_lock->owner() != NULL,
+ "Should be owned on this thread's behalf.");
+ size_t result = _summary_bytes_used;
+ if (_cur_alloc_region != NULL)
+ result += _cur_alloc_region->used();
+ return result;
+}
+
+class SumUsedClosure: public HeapRegionClosure {
+ size_t _used;
+public:
+ SumUsedClosure() : _used(0) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->continuesHumongous()) {
+ _used += r->used();
+ }
+ return false;
+ }
+ size_t result() { return _used; }
+};
+
+size_t G1CollectedHeap::recalculate_used() const {
+ SumUsedClosure blk;
+ _hrs->iterate(&blk);
+ return blk.result();
+}
+
+#ifndef PRODUCT
+class SumUsedRegionsClosure: public HeapRegionClosure {
+ size_t _num;
+public:
+ // _num is set to 1 to account for the popular region
+ SumUsedRegionsClosure() : _num(G1NumPopularRegions) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->continuesHumongous() || r->used() > 0 || r->is_gc_alloc_region()) {
+ _num += 1;
+ }
+ return false;
+ }
+ size_t result() { return _num; }
+};
+
+size_t G1CollectedHeap::recalculate_used_regions() const {
+ SumUsedRegionsClosure blk;
+ _hrs->iterate(&blk);
+ return blk.result();
+}
+#endif // PRODUCT
+
+size_t G1CollectedHeap::unsafe_max_alloc() {
+ if (_free_regions > 0) return HeapRegion::GrainBytes;
+ // otherwise, is there space in the current allocation region?
+
+ // We need to store the current allocation region in a local variable
+ // here. The problem is that this method doesn't take any locks and
+ // there may be other threads which overwrite the current allocation
+ // region field. attempt_allocation(), for example, sets it to NULL
+ // and this can happen *after* the NULL check here but before the call
+ // to free(), resulting in a SIGSEGV. Note that this doesn't appear
+ // to be a problem in the optimized build, since the two loads of the
+ // current allocation region field are optimized away.
+ HeapRegion* car = _cur_alloc_region;
+
+ // FIXME: should iterate over all regions?
+ if (car == NULL) {
+ return 0;
+ }
+ return car->free();
+}
+
+void G1CollectedHeap::collect(GCCause::Cause cause) {
+ // The caller doesn't have the Heap_lock
+ assert(!Heap_lock->owned_by_self(), "this thread should not own the Heap_lock");
+ MutexLocker ml(Heap_lock);
+ collect_locked(cause);
+}
+
+void G1CollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {
+ assert(Thread::current()->is_VM_thread(), "Precondition#1");
+ assert(Heap_lock->is_locked(), "Precondition#2");
+ GCCauseSetter gcs(this, cause);
+ switch (cause) {
+ case GCCause::_heap_inspection:
+ case GCCause::_heap_dump: {
+ HandleMark hm;
+ do_full_collection(false); // don't clear all soft refs
+ break;
+ }
+ default: // XXX FIX ME
+ ShouldNotReachHere(); // Unexpected use of this function
+ }
+}
+
+
+void G1CollectedHeap::collect_locked(GCCause::Cause cause) {
+ // Don't want to do a GC until cleanup is completed.
+ wait_for_cleanup_complete();
+
+ // Read the GC count while holding the Heap_lock
+ int gc_count_before = SharedHeap::heap()->total_collections();
+ {
+ MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back
+ VM_G1CollectFull op(gc_count_before, cause);
+ VMThread::execute(&op);
+ }
+}
+
+bool G1CollectedHeap::is_in(const void* p) const {
+ if (_g1_committed.contains(p)) {
+ HeapRegion* hr = _hrs->addr_to_region(p);
+ return hr->is_in(p);
+ } else {
+ return _perm_gen->as_gen()->is_in(p);
+ }
+}
+
+// Iteration functions.
+
+// Iterates an OopClosure over all ref-containing fields of objects
+// within a HeapRegion.
+
+class IterateOopClosureRegionClosure: public HeapRegionClosure {
+ MemRegion _mr;
+ OopClosure* _cl;
+public:
+ IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)
+ : _mr(mr), _cl(cl) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (! r->continuesHumongous()) {
+ r->oop_iterate(_cl);
+ }
+ return false;
+ }
+};
+
+void G1CollectedHeap::oop_iterate(OopClosure* cl) {
+ IterateOopClosureRegionClosure blk(_g1_committed, cl);
+ _hrs->iterate(&blk);
+}
+
+void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl) {
+ IterateOopClosureRegionClosure blk(mr, cl);
+ _hrs->iterate(&blk);
+}
+
+// Iterates an ObjectClosure over all objects within a HeapRegion.
+
+class IterateObjectClosureRegionClosure: public HeapRegionClosure {
+ ObjectClosure* _cl;
+public:
+ IterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (! r->continuesHumongous()) {
+ r->object_iterate(_cl);
+ }
+ return false;
+ }
+};
+
+void G1CollectedHeap::object_iterate(ObjectClosure* cl) {
+ IterateObjectClosureRegionClosure blk(cl);
+ _hrs->iterate(&blk);
+}
+
+void G1CollectedHeap::object_iterate_since_last_GC(ObjectClosure* cl) {
+ // FIXME: is this right?
+ guarantee(false, "object_iterate_since_last_GC not supported by G1 heap");
+}
+
+// Calls a SpaceClosure on a HeapRegion.
+
+class SpaceClosureRegionClosure: public HeapRegionClosure {
+ SpaceClosure* _cl;
+public:
+ SpaceClosureRegionClosure(SpaceClosure* cl) : _cl(cl) {}
+ bool doHeapRegion(HeapRegion* r) {
+ _cl->do_space(r);
+ return false;
+ }
+};
+
+void G1CollectedHeap::space_iterate(SpaceClosure* cl) {
+ SpaceClosureRegionClosure blk(cl);
+ _hrs->iterate(&blk);
+}
+
+void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {
+ _hrs->iterate(cl);
+}
+
+void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,
+ HeapRegionClosure* cl) {
+ _hrs->iterate_from(r, cl);
+}
+
+void
+G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {
+ _hrs->iterate_from(idx, cl);
+}
+
+HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }
+
+void
+G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,
+ int worker,
+ jint claim_value) {
+ const size_t regions = n_regions();
+ const size_t worker_num = (ParallelGCThreads > 0 ? ParallelGCThreads : 1);
+ // try to spread out the starting points of the workers
+ const size_t start_index = regions / worker_num * (size_t) worker;
+
+ // each worker will actually look at all regions
+ for (size_t count = 0; count < regions; ++count) {
+ const size_t index = (start_index + count) % regions;
+ assert(0 <= index && index < regions, "sanity");
+ HeapRegion* r = region_at(index);
+ // we'll ignore "continues humongous" regions (we'll process them
+ // when we come across their corresponding "start humongous"
+ // region) and regions already claimed
+ if (r->claim_value() == claim_value || r->continuesHumongous()) {
+ continue;
+ }
+ // OK, try to claim it
+ if (r->claimHeapRegion(claim_value)) {
+ // success!
+ assert(!r->continuesHumongous(), "sanity");
+ if (r->startsHumongous()) {
+ // If the region is "starts humongous" we'll iterate over its
+ // "continues humongous" first; in fact we'll do them
+ // first. The order is important. In on case, calling the
+ // closure on the "starts humongous" region might de-allocate
+ // and clear all its "continues humongous" regions and, as a
+ // result, we might end up processing them twice. So, we'll do
+ // them first (notice: most closures will ignore them anyway) and
+ // then we'll do the "starts humongous" region.
+ for (size_t ch_index = index + 1; ch_index < regions; ++ch_index) {
+ HeapRegion* chr = region_at(ch_index);
+
+ // if the region has already been claimed or it's not
+ // "continues humongous" we're done
+ if (chr->claim_value() == claim_value ||
+ !chr->continuesHumongous()) {
+ break;
+ }
+
+ // Noone should have claimed it directly. We can given
+ // that we claimed its "starts humongous" region.
+ assert(chr->claim_value() != claim_value, "sanity");
+ assert(chr->humongous_start_region() == r, "sanity");
+
+ if (chr->claimHeapRegion(claim_value)) {
+ // we should always be able to claim it; noone else should
+ // be trying to claim this region
+
+ bool res2 = cl->doHeapRegion(chr);
+ assert(!res2, "Should not abort");
+
+ // Right now, this holds (i.e., no closure that actually
+ // does something with "continues humongous" regions
+ // clears them). We might have to weaken it in the future,
+ // but let's leave these two asserts here for extra safety.
+ assert(chr->continuesHumongous(), "should still be the case");
+ assert(chr->humongous_start_region() == r, "sanity");
+ } else {
+ guarantee(false, "we should not reach here");
+ }
+ }
+ }
+
+ assert(!r->continuesHumongous(), "sanity");
+ bool res = cl->doHeapRegion(r);
+ assert(!res, "Should not abort");
+ }
+ }
+}
+
+class ResetClaimValuesClosure: public HeapRegionClosure {
+public:
+ bool doHeapRegion(HeapRegion* r) {
+ r->set_claim_value(HeapRegion::InitialClaimValue);
+ return false;
+ }
+};
+
+void
+G1CollectedHeap::reset_heap_region_claim_values() {
+ ResetClaimValuesClosure blk;
+ heap_region_iterate(&blk);
+}
+
+#ifdef ASSERT
+// This checks whether all regions in the heap have the correct claim
+// value. I also piggy-backed on this a check to ensure that the
+// humongous_start_region() information on "continues humongous"
+// regions is correct.
+
+class CheckClaimValuesClosure : public HeapRegionClosure {
+private:
+ jint _claim_value;
+ size_t _failures;
+ HeapRegion* _sh_region;
+public:
+ CheckClaimValuesClosure(jint claim_value) :
+ _claim_value(claim_value), _failures(0), _sh_region(NULL) { }
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->claim_value() != _claim_value) {
+ gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "
+ "claim value = %d, should be %d",
+ r->bottom(), r->end(), r->claim_value(),
+ _claim_value);
+ ++_failures;
+ }
+ if (!r->isHumongous()) {
+ _sh_region = NULL;
+ } else if (r->startsHumongous()) {
+ _sh_region = r;
+ } else if (r->continuesHumongous()) {
+ if (r->humongous_start_region() != _sh_region) {
+ gclog_or_tty->print_cr("Region ["PTR_FORMAT","PTR_FORMAT"), "
+ "HS = "PTR_FORMAT", should be "PTR_FORMAT,
+ r->bottom(), r->end(),
+ r->humongous_start_region(),
+ _sh_region);
+ ++_failures;
+ }
+ }
+ return false;
+ }
+ size_t failures() {
+ return _failures;
+ }
+};
+
+bool G1CollectedHeap::check_heap_region_claim_values(jint claim_value) {
+ CheckClaimValuesClosure cl(claim_value);
+ heap_region_iterate(&cl);
+ return cl.failures() == 0;
+}
+#endif // ASSERT
+
+void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {
+ HeapRegion* r = g1_policy()->collection_set();
+ while (r != NULL) {
+ HeapRegion* next = r->next_in_collection_set();
+ if (cl->doHeapRegion(r)) {
+ cl->incomplete();
+ return;
+ }
+ r = next;
+ }
+}
+
+void G1CollectedHeap::collection_set_iterate_from(HeapRegion* r,
+ HeapRegionClosure *cl) {
+ assert(r->in_collection_set(),
+ "Start region must be a member of the collection set.");
+ HeapRegion* cur = r;
+ while (cur != NULL) {
+ HeapRegion* next = cur->next_in_collection_set();
+ if (cl->doHeapRegion(cur) && false) {
+ cl->incomplete();
+ return;
+ }
+ cur = next;
+ }
+ cur = g1_policy()->collection_set();
+ while (cur != r) {
+ HeapRegion* next = cur->next_in_collection_set();
+ if (cl->doHeapRegion(cur) && false) {
+ cl->incomplete();
+ return;
+ }
+ cur = next;
+ }
+}
+
+CompactibleSpace* G1CollectedHeap::first_compactible_space() {
+ return _hrs->length() > 0 ? _hrs->at(0) : NULL;
+}
+
+
+Space* G1CollectedHeap::space_containing(const void* addr) const {
+ Space* res = heap_region_containing(addr);
+ if (res == NULL)
+ res = perm_gen()->space_containing(addr);
+ return res;
+}
+
+HeapWord* G1CollectedHeap::block_start(const void* addr) const {
+ Space* sp = space_containing(addr);
+ if (sp != NULL) {
+ return sp->block_start(addr);
+ }
+ return NULL;
+}
+
+size_t G1CollectedHeap::block_size(const HeapWord* addr) const {
+ Space* sp = space_containing(addr);
+ assert(sp != NULL, "block_size of address outside of heap");
+ return sp->block_size(addr);
+}
+
+bool G1CollectedHeap::block_is_obj(const HeapWord* addr) const {
+ Space* sp = space_containing(addr);
+ return sp->block_is_obj(addr);
+}
+
+bool G1CollectedHeap::supports_tlab_allocation() const {
+ return true;
+}
+
+size_t G1CollectedHeap::tlab_capacity(Thread* ignored) const {
+ return HeapRegion::GrainBytes;
+}
+
+size_t G1CollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {
+ // Return the remaining space in the cur alloc region, but not less than
+ // the min TLAB size.
+ // Also, no more than half the region size, since we can't allow tlabs to
+ // grow big enough to accomodate humongous objects.
+
+ // We need to story it locally, since it might change between when we
+ // test for NULL and when we use it later.
+ ContiguousSpace* cur_alloc_space = _cur_alloc_region;
+ if (cur_alloc_space == NULL) {
+ return HeapRegion::GrainBytes/2;
+ } else {
+ return MAX2(MIN2(cur_alloc_space->free(),
+ (size_t)(HeapRegion::GrainBytes/2)),
+ (size_t)MinTLABSize);
+ }
+}
+
+HeapWord* G1CollectedHeap::allocate_new_tlab(size_t size) {
+ bool dummy;
+ return G1CollectedHeap::mem_allocate(size, false, true, &dummy);
+}
+
+bool G1CollectedHeap::allocs_are_zero_filled() {
+ return false;
+}
+
+size_t G1CollectedHeap::large_typearray_limit() {
+ // FIXME
+ return HeapRegion::GrainBytes/HeapWordSize;
+}
+
+size_t G1CollectedHeap::max_capacity() const {
+ return _g1_committed.byte_size();
+}
+
+jlong G1CollectedHeap::millis_since_last_gc() {
+ // assert(false, "NYI");
+ return 0;
+}
+
+
+void G1CollectedHeap::prepare_for_verify() {
+ if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
+ ensure_parsability(false);
+ }
+ g1_rem_set()->prepare_for_verify();
+}
+
+class VerifyLivenessOopClosure: public OopClosure {
+ G1CollectedHeap* g1h;
+public:
+ VerifyLivenessOopClosure(G1CollectedHeap* _g1h) {
+ g1h = _g1h;
+ }
+ void do_oop(narrowOop *p) {
+ guarantee(false, "NYI");
+ }
+ void do_oop(oop *p) {
+ oop obj = *p;
+ assert(obj == NULL || !g1h->is_obj_dead(obj),
+ "Dead object referenced by a not dead object");
+ }
+};
+
+class VerifyObjsInRegionClosure: public ObjectClosure {
+ G1CollectedHeap* _g1h;
+ size_t _live_bytes;
+ HeapRegion *_hr;
+public:
+ VerifyObjsInRegionClosure(HeapRegion *hr) : _live_bytes(0), _hr(hr) {
+ _g1h = G1CollectedHeap::heap();
+ }
+ void do_object(oop o) {
+ VerifyLivenessOopClosure isLive(_g1h);
+ assert(o != NULL, "Huh?");
+ if (!_g1h->is_obj_dead(o)) {
+ o->oop_iterate(&isLive);
+ if (!_hr->obj_allocated_since_prev_marking(o))
+ _live_bytes += (o->size() * HeapWordSize);
+ }
+ }
+ size_t live_bytes() { return _live_bytes; }
+};
+
+class PrintObjsInRegionClosure : public ObjectClosure {
+ HeapRegion *_hr;
+ G1CollectedHeap *_g1;
+public:
+ PrintObjsInRegionClosure(HeapRegion *hr) : _hr(hr) {
+ _g1 = G1CollectedHeap::heap();
+ };
+
+ void do_object(oop o) {
+ if (o != NULL) {
+ HeapWord *start = (HeapWord *) o;
+ size_t word_sz = o->size();
+ gclog_or_tty->print("\nPrinting obj "PTR_FORMAT" of size " SIZE_FORMAT
+ " isMarkedPrev %d isMarkedNext %d isAllocSince %d\n",
+ (void*) o, word_sz,
+ _g1->isMarkedPrev(o),
+ _g1->isMarkedNext(o),
+ _hr->obj_allocated_since_prev_marking(o));
+ HeapWord *end = start + word_sz;
+ HeapWord *cur;
+ int *val;
+ for (cur = start; cur < end; cur++) {
+ val = (int *) cur;
+ gclog_or_tty->print("\t "PTR_FORMAT":"PTR_FORMAT"\n", val, *val);
+ }
+ }
+ }
+};
+
+class VerifyRegionClosure: public HeapRegionClosure {
+public:
+ bool _allow_dirty;
+ bool _par;
+ VerifyRegionClosure(bool allow_dirty, bool par = false)
+ : _allow_dirty(allow_dirty), _par(par) {}
+ bool doHeapRegion(HeapRegion* r) {
+ guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
+ "Should be unclaimed at verify points.");
+ if (r->isHumongous()) {
+ if (r->startsHumongous()) {
+ // Verify the single H object.
+ oop(r->bottom())->verify();
+ size_t word_sz = oop(r->bottom())->size();
+ guarantee(r->top() == r->bottom() + word_sz,
+ "Only one object in a humongous region");
+ }
+ } else {
+ VerifyObjsInRegionClosure not_dead_yet_cl(r);
+ r->verify(_allow_dirty);
+ r->object_iterate(¬_dead_yet_cl);
+ guarantee(r->max_live_bytes() >= not_dead_yet_cl.live_bytes(),
+ "More live objects than counted in last complete marking.");
+ }
+ return false;
+ }
+};
+
+class VerifyRootsClosure: public OopsInGenClosure {
+private:
+ G1CollectedHeap* _g1h;
+ bool _failures;
+
+public:
+ VerifyRootsClosure() :
+ _g1h(G1CollectedHeap::heap()), _failures(false) { }
+
+ bool failures() { return _failures; }
+
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+
+ void do_oop(oop* p) {
+ oop obj = *p;
+ if (obj != NULL) {
+ if (_g1h->is_obj_dead(obj)) {
+ gclog_or_tty->print_cr("Root location "PTR_FORMAT" "
+ "points to dead obj "PTR_FORMAT, p, (void*) obj);
+ obj->print_on(gclog_or_tty);
+ _failures = true;
+ }
+ }
+ }
+};
+
+// This is the task used for parallel heap verification.
+
+class G1ParVerifyTask: public AbstractGangTask {
+private:
+ G1CollectedHeap* _g1h;
+ bool _allow_dirty;
+
+public:
+ G1ParVerifyTask(G1CollectedHeap* g1h, bool allow_dirty) :
+ AbstractGangTask("Parallel verify task"),
+ _g1h(g1h), _allow_dirty(allow_dirty) { }
+
+ void work(int worker_i) {
+ VerifyRegionClosure blk(_allow_dirty, true);
+ _g1h->heap_region_par_iterate_chunked(&blk, worker_i,
+ HeapRegion::ParVerifyClaimValue);
+ }
+};
+
+void G1CollectedHeap::verify(bool allow_dirty, bool silent) {
+ if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
+ if (!silent) { gclog_or_tty->print("roots "); }
+ VerifyRootsClosure rootsCl;
+ process_strong_roots(false,
+ SharedHeap::SO_AllClasses,
+ &rootsCl,
+ &rootsCl);
+ rem_set()->invalidate(perm_gen()->used_region(), false);
+ if (!silent) { gclog_or_tty->print("heapRegions "); }
+ if (GCParallelVerificationEnabled && ParallelGCThreads > 1) {
+ assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),
+ "sanity check");
+
+ G1ParVerifyTask task(this, allow_dirty);
+ int n_workers = workers()->total_workers();
+ set_par_threads(n_workers);
+ workers()->run_task(&task);
+ set_par_threads(0);
+
+ assert(check_heap_region_claim_values(HeapRegion::ParVerifyClaimValue),
+ "sanity check");
+
+ reset_heap_region_claim_values();
+
+ assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),
+ "sanity check");
+ } else {
+ VerifyRegionClosure blk(allow_dirty);
+ _hrs->iterate(&blk);
+ }
+ if (!silent) gclog_or_tty->print("remset ");
+ rem_set()->verify();
+ guarantee(!rootsCl.failures(), "should not have had failures");
+ } else {
+ if (!silent) gclog_or_tty->print("(SKIPPING roots, heapRegions, remset) ");
+ }
+}
+
+class PrintRegionClosure: public HeapRegionClosure {
+ outputStream* _st;
+public:
+ PrintRegionClosure(outputStream* st) : _st(st) {}
+ bool doHeapRegion(HeapRegion* r) {
+ r->print_on(_st);
+ return false;
+ }
+};
+
+void G1CollectedHeap::print() const { print_on(gclog_or_tty); }
+
+void G1CollectedHeap::print_on(outputStream* st) const {
+ PrintRegionClosure blk(st);
+ _hrs->iterate(&blk);
+}
+
+void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {
+ if (ParallelGCThreads > 0) {
+ workers()->print_worker_threads();
+ }
+ st->print("\"G1 concurrent mark GC Thread\" ");
+ _cmThread->print();
+ st->cr();
+ st->print("\"G1 concurrent refinement GC Thread\" ");
+ _cg1r->cg1rThread()->print_on(st);
+ st->cr();
+ st->print("\"G1 zero-fill GC Thread\" ");
+ _czft->print_on(st);
+ st->cr();
+}
+
+void G1CollectedHeap::gc_threads_do(ThreadClosure* tc) const {
+ if (ParallelGCThreads > 0) {
+ workers()->threads_do(tc);
+ }
+ tc->do_thread(_cmThread);
+ tc->do_thread(_cg1r->cg1rThread());
+ tc->do_thread(_czft);
+}
+
+void G1CollectedHeap::print_tracing_info() const {
+ concurrent_g1_refine()->print_final_card_counts();
+
+ // We'll overload this to mean "trace GC pause statistics."
+ if (TraceGen0Time || TraceGen1Time) {
+ // The "G1CollectorPolicy" is keeping track of these stats, so delegate
+ // to that.
+ g1_policy()->print_tracing_info();
+ }
+ if (SummarizeG1RSStats) {
+ g1_rem_set()->print_summary_info();
+ }
+ if (SummarizeG1ConcMark) {
+ concurrent_mark()->print_summary_info();
+ }
+ if (SummarizeG1ZFStats) {
+ ConcurrentZFThread::print_summary_info();
+ }
+ if (G1SummarizePopularity) {
+ print_popularity_summary_info();
+ }
+ g1_policy()->print_yg_surv_rate_info();
+
+ GCOverheadReporter::printGCOverhead();
+
+ SpecializationStats::print();
+}
+
+
+int G1CollectedHeap::addr_to_arena_id(void* addr) const {
+ HeapRegion* hr = heap_region_containing(addr);
+ if (hr == NULL) {
+ return 0;
+ } else {
+ return 1;
+ }
+}
+
+G1CollectedHeap* G1CollectedHeap::heap() {
+ assert(_sh->kind() == CollectedHeap::G1CollectedHeap,
+ "not a garbage-first heap");
+ return _g1h;
+}
+
+void G1CollectedHeap::gc_prologue(bool full /* Ignored */) {
+ if (PrintHeapAtGC){
+ gclog_or_tty->print_cr(" {Heap before GC collections=%d:", total_collections());
+ Universe::print();
+ }
+ assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
+ // Call allocation profiler
+ AllocationProfiler::iterate_since_last_gc();
+ // Fill TLAB's and such
+ ensure_parsability(true);
+}
+
+void G1CollectedHeap::gc_epilogue(bool full /* Ignored */) {
+ // FIXME: what is this about?
+ // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
+ // is set.
+ COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),
+ "derived pointer present"));
+
+ if (PrintHeapAtGC){
+ gclog_or_tty->print_cr(" Heap after GC collections=%d:", total_collections());
+ Universe::print();
+ gclog_or_tty->print("} ");
+ }
+}
+
+void G1CollectedHeap::do_collection_pause() {
+ // Read the GC count while holding the Heap_lock
+ // we need to do this _before_ wait_for_cleanup_complete(), to
+ // ensure that we do not give up the heap lock and potentially
+ // pick up the wrong count
+ int gc_count_before = SharedHeap::heap()->total_collections();
+
+ // Don't want to do a GC pause while cleanup is being completed!
+ wait_for_cleanup_complete();
+
+ g1_policy()->record_stop_world_start();
+ {
+ MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back
+ VM_G1IncCollectionPause op(gc_count_before);
+ VMThread::execute(&op);
+ }
+}
+
+void
+G1CollectedHeap::doConcurrentMark() {
+ if (G1ConcMark) {
+ MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);
+ if (!_cmThread->in_progress()) {
+ _cmThread->set_started();
+ CGC_lock->notify();
+ }
+ }
+}
+
+class VerifyMarkedObjsClosure: public ObjectClosure {
+ G1CollectedHeap* _g1h;
+ public:
+ VerifyMarkedObjsClosure(G1CollectedHeap* g1h) : _g1h(g1h) {}
+ void do_object(oop obj) {
+ assert(obj->mark()->is_marked() ? !_g1h->is_obj_dead(obj) : true,
+ "markandsweep mark should agree with concurrent deadness");
+ }
+};
+
+void
+G1CollectedHeap::checkConcurrentMark() {
+ VerifyMarkedObjsClosure verifycl(this);
+ doConcurrentMark();
+ // MutexLockerEx x(getMarkBitMapLock(),
+ // Mutex::_no_safepoint_check_flag);
+ object_iterate(&verifycl);
+}
+
+void G1CollectedHeap::do_sync_mark() {
+ _cm->checkpointRootsInitial();
+ _cm->markFromRoots();
+ _cm->checkpointRootsFinal(false);
+}
+
+//
+
+double G1CollectedHeap::predict_region_elapsed_time_ms(HeapRegion *hr,
+ bool young) {
+ return _g1_policy->predict_region_elapsed_time_ms(hr, young);
+}
+
+void G1CollectedHeap::check_if_region_is_too_expensive(double
+ predicted_time_ms) {
+ _g1_policy->check_if_region_is_too_expensive(predicted_time_ms);
+}
+
+size_t G1CollectedHeap::pending_card_num() {
+ size_t extra_cards = 0;
+ JavaThread *curr = Threads::first();
+ while (curr != NULL) {
+ DirtyCardQueue& dcq = curr->dirty_card_queue();
+ extra_cards += dcq.size();
+ curr = curr->next();
+ }
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ size_t buffer_size = dcqs.buffer_size();
+ size_t buffer_num = dcqs.completed_buffers_num();
+ return buffer_size * buffer_num + extra_cards;
+}
+
+size_t G1CollectedHeap::max_pending_card_num() {
+ DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
+ size_t buffer_size = dcqs.buffer_size();
+ size_t buffer_num = dcqs.completed_buffers_num();
+ int thread_num = Threads::number_of_threads();
+ return (buffer_num + thread_num) * buffer_size;
+}
+
+size_t G1CollectedHeap::cards_scanned() {
+ HRInto_G1RemSet* g1_rset = (HRInto_G1RemSet*) g1_rem_set();
+ return g1_rset->cardsScanned();
+}
+
+void
+G1CollectedHeap::setup_surviving_young_words() {
+ guarantee( _surviving_young_words == NULL, "pre-condition" );
+ size_t array_length = g1_policy()->young_cset_length();
+ _surviving_young_words = NEW_C_HEAP_ARRAY(size_t, array_length);
+ if (_surviving_young_words == NULL) {
+ vm_exit_out_of_memory(sizeof(size_t) * array_length,
+ "Not enough space for young surv words summary.");
+ }
+ memset(_surviving_young_words, 0, array_length * sizeof(size_t));
+ for (size_t i = 0; i < array_length; ++i) {
+ guarantee( _surviving_young_words[i] == 0, "invariant" );
+ }
+}
+
+void
+G1CollectedHeap::update_surviving_young_words(size_t* surv_young_words) {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ size_t array_length = g1_policy()->young_cset_length();
+ for (size_t i = 0; i < array_length; ++i)
+ _surviving_young_words[i] += surv_young_words[i];
+}
+
+void
+G1CollectedHeap::cleanup_surviving_young_words() {
+ guarantee( _surviving_young_words != NULL, "pre-condition" );
+ FREE_C_HEAP_ARRAY(size_t, _surviving_young_words);
+ _surviving_young_words = NULL;
+}
+
+//
+
+void
+G1CollectedHeap::do_collection_pause_at_safepoint(HeapRegion* popular_region) {
+ char verbose_str[128];
+ sprintf(verbose_str, "GC pause ");
+ if (popular_region != NULL)
+ strcat(verbose_str, "(popular)");
+ else if (g1_policy()->in_young_gc_mode()) {
+ if (g1_policy()->full_young_gcs())
+ strcat(verbose_str, "(young)");
+ else
+ strcat(verbose_str, "(partial)");
+ }
+ bool reset_should_initiate_conc_mark = false;
+ if (popular_region != NULL && g1_policy()->should_initiate_conc_mark()) {
+ // we currently do not allow an initial mark phase to be piggy-backed
+ // on a popular pause
+ reset_should_initiate_conc_mark = true;
+ g1_policy()->unset_should_initiate_conc_mark();
+ }
+ if (g1_policy()->should_initiate_conc_mark())
+ strcat(verbose_str, " (initial-mark)");
+
+ GCCauseSetter x(this, (popular_region == NULL ?
+ GCCause::_g1_inc_collection_pause :
+ GCCause::_g1_pop_region_collection_pause));
+
+ // if PrintGCDetails is on, we'll print long statistics information
+ // in the collector policy code, so let's not print this as the output
+ // is messy if we do.
+ gclog_or_tty->date_stamp(PrintGC && PrintGCDateStamps);
+ TraceCPUTime tcpu(PrintGCDetails, true, gclog_or_tty);
+ TraceTime t(verbose_str, PrintGC && !PrintGCDetails, true, gclog_or_tty);
+
+ ResourceMark rm;
+ assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");
+ assert(Thread::current() == VMThread::vm_thread(), "should be in vm thread");
+ guarantee(!is_gc_active(), "collection is not reentrant");
+ assert(regions_accounted_for(), "Region leakage!");
+
+ increment_gc_time_stamp();
+
+ if (g1_policy()->in_young_gc_mode()) {
+ assert(check_young_list_well_formed(),
+ "young list should be well formed");
+ }
+
+ if (GC_locker::is_active()) {
+ return; // GC is disabled (e.g. JNI GetXXXCritical operation)
+ }
+
+ bool abandoned = false;
+ { // Call to jvmpi::post_class_unload_events must occur outside of active GC
+ IsGCActiveMark x;
+
+ gc_prologue(false);
+ increment_total_collections();
+
+#if G1_REM_SET_LOGGING
+ gclog_or_tty->print_cr("\nJust chose CS, heap:");
+ print();
+#endif
+
+ if (VerifyBeforeGC && total_collections() >= VerifyGCStartAt) {
+ HandleMark hm; // Discard invalid handles created during verification
+ prepare_for_verify();
+ gclog_or_tty->print(" VerifyBeforeGC:");
+ Universe::verify(false);
+ }
+
+ COMPILER2_PRESENT(DerivedPointerTable::clear());
+
+ // We want to turn off ref discovery, if necessary, and turn it back on
+ // on again later if we do.
+ bool was_enabled = ref_processor()->discovery_enabled();
+ if (was_enabled) ref_processor()->disable_discovery();
+
+ // Forget the current alloc region (we might even choose it to be part
+ // of the collection set!).
+ abandon_cur_alloc_region();
+
+ // The elapsed time induced by the start time below deliberately elides
+ // the possible verification above.
+ double start_time_sec = os::elapsedTime();
+ GCOverheadReporter::recordSTWStart(start_time_sec);
+ size_t start_used_bytes = used();
+ if (!G1ConcMark) {
+ do_sync_mark();
+ }
+
+ g1_policy()->record_collection_pause_start(start_time_sec,
+ start_used_bytes);
+
+#if SCAN_ONLY_VERBOSE
+ _young_list->print();
+#endif // SCAN_ONLY_VERBOSE
+
+ if (g1_policy()->should_initiate_conc_mark()) {
+ concurrent_mark()->checkpointRootsInitialPre();
+ }
+ save_marks();
+
+ // We must do this before any possible evacuation that should propogate
+ // marks, including evacuation of popular objects in a popular pause.
+ if (mark_in_progress()) {
+ double start_time_sec = os::elapsedTime();
+
+ _cm->drainAllSATBBuffers();
+ double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
+ g1_policy()->record_satb_drain_time(finish_mark_ms);
+
+ }
+ // Record the number of elements currently on the mark stack, so we
+ // only iterate over these. (Since evacuation may add to the mark
+ // stack, doing more exposes race conditions.) If no mark is in
+ // progress, this will be zero.
+ _cm->set_oops_do_bound();
+
+ assert(regions_accounted_for(), "Region leakage.");
+
+ bool abandoned = false;
+
+ if (mark_in_progress())
+ concurrent_mark()->newCSet();
+
+ // Now choose the CS.
+ if (popular_region == NULL) {
+ g1_policy()->choose_collection_set();
+ } else {
+ // We may be evacuating a single region (for popularity).
+ g1_policy()->record_popular_pause_preamble_start();
+ popularity_pause_preamble(popular_region);
+ g1_policy()->record_popular_pause_preamble_end();
+ abandoned = (g1_policy()->collection_set() == NULL);
+ // Now we allow more regions to be added (we have to collect
+ // all popular regions).
+ if (!abandoned) {
+ g1_policy()->choose_collection_set(popular_region);
+ }
+ }
+ // We may abandon a pause if we find no region that will fit in the MMU
+ // pause.
+ abandoned = (g1_policy()->collection_set() == NULL);
+
+ // Nothing to do if we were unable to choose a collection set.
+ if (!abandoned) {
+#if G1_REM_SET_LOGGING
+ gclog_or_tty->print_cr("\nAfter pause, heap:");
+ print();
+#endif
+
+ setup_surviving_young_words();
+
+ // Set up the gc allocation regions.
+ get_gc_alloc_regions();
+
+ // Actually do the work...
+ evacuate_collection_set();
+ free_collection_set(g1_policy()->collection_set());
+ g1_policy()->clear_collection_set();
+
+ if (popular_region != NULL) {
+ // We have to wait until now, because we don't want the region to
+ // be rescheduled for pop-evac during RS update.
+ popular_region->set_popular_pending(false);
+ }
+
+ release_gc_alloc_regions();
+
+ cleanup_surviving_young_words();
+
+ if (g1_policy()->in_young_gc_mode()) {
+ _young_list->reset_sampled_info();
+ assert(check_young_list_empty(true),
+ "young list should be empty");
+
+#if SCAN_ONLY_VERBOSE
+ _young_list->print();
+#endif // SCAN_ONLY_VERBOSE
+
+ _young_list->reset_auxilary_lists();
+ }
+ } else {
+ COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+ }
+
+ if (evacuation_failed()) {
+ _summary_bytes_used = recalculate_used();
+ } else {
+ // The "used" of the the collection set have already been subtracted
+ // when they were freed. Add in the bytes evacuated.
+ _summary_bytes_used += g1_policy()->bytes_in_to_space();
+ }
+
+ if (g1_policy()->in_young_gc_mode() &&
+ g1_policy()->should_initiate_conc_mark()) {
+ concurrent_mark()->checkpointRootsInitialPost();
+ set_marking_started();
+ doConcurrentMark();
+ }
+
+#if SCAN_ONLY_VERBOSE
+ _young_list->print();
+#endif // SCAN_ONLY_VERBOSE
+
+ double end_time_sec = os::elapsedTime();
+ g1_policy()->record_pause_time((end_time_sec - start_time_sec)*1000.0);
+ GCOverheadReporter::recordSTWEnd(end_time_sec);
+ g1_policy()->record_collection_pause_end(popular_region != NULL,
+ abandoned);
+
+ assert(regions_accounted_for(), "Region leakage.");
+
+ if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
+ HandleMark hm; // Discard invalid handles created during verification
+ gclog_or_tty->print(" VerifyAfterGC:");
+ Universe::verify(false);
+ }
+
+ if (was_enabled) ref_processor()->enable_discovery();
+
+ {
+ size_t expand_bytes = g1_policy()->expansion_amount();
+ if (expand_bytes > 0) {
+ size_t bytes_before = capacity();
+ expand(expand_bytes);
+ }
+ }
+
+ if (mark_in_progress())
+ concurrent_mark()->update_g1_committed();
+
+ gc_epilogue(false);
+ }
+
+ assert(verify_region_lists(), "Bad region lists.");
+
+ if (reset_should_initiate_conc_mark)
+ g1_policy()->set_should_initiate_conc_mark();
+
+ if (ExitAfterGCNum > 0 && total_collections() == ExitAfterGCNum) {
+ gclog_or_tty->print_cr("Stopping after GC #%d", ExitAfterGCNum);
+ print_tracing_info();
+ vm_exit(-1);
+ }
+}
+
+void G1CollectedHeap::set_gc_alloc_region(int purpose, HeapRegion* r) {
+ assert(purpose >= 0 && purpose < GCAllocPurposeCount, "invalid purpose");
+ HeapWord* original_top = NULL;
+ if (r != NULL)
+ original_top = r->top();
+
+ // We will want to record the used space in r as being there before gc.
+ // One we install it as a GC alloc region it's eligible for allocation.
+ // So record it now and use it later.
+ size_t r_used = 0;
+ if (r != NULL) {
+ r_used = r->used();
+
+ if (ParallelGCThreads > 0) {
+ // need to take the lock to guard against two threads calling
+ // get_gc_alloc_region concurrently (very unlikely but...)
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ r->save_marks();
+ }
+ }
+ HeapRegion* old_alloc_region = _gc_alloc_regions[purpose];
+ _gc_alloc_regions[purpose] = r;
+ if (old_alloc_region != NULL) {
+ // Replace aliases too.
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ if (_gc_alloc_regions[ap] == old_alloc_region) {
+ _gc_alloc_regions[ap] = r;
+ }
+ }
+ }
+ if (r != NULL) {
+ push_gc_alloc_region(r);
+ if (mark_in_progress() && original_top != r->next_top_at_mark_start()) {
+ // We are using a region as a GC alloc region after it has been used
+ // as a mutator allocation region during the current marking cycle.
+ // The mutator-allocated objects are currently implicitly marked, but
+ // when we move hr->next_top_at_mark_start() forward at the the end
+ // of the GC pause, they won't be. We therefore mark all objects in
+ // the "gap". We do this object-by-object, since marking densely
+ // does not currently work right with marking bitmap iteration. This
+ // means we rely on TLAB filling at the start of pauses, and no
+ // "resuscitation" of filled TLAB's. If we want to do this, we need
+ // to fix the marking bitmap iteration.
+ HeapWord* curhw = r->next_top_at_mark_start();
+ HeapWord* t = original_top;
+
+ while (curhw < t) {
+ oop cur = (oop)curhw;
+ // We'll assume parallel for generality. This is rare code.
+ concurrent_mark()->markAndGrayObjectIfNecessary(cur); // can't we just mark them?
+ curhw = curhw + cur->size();
+ }
+ assert(curhw == t, "Should have parsed correctly.");
+ }
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print("New alloc region ["PTR_FORMAT", "PTR_FORMAT", " PTR_FORMAT") "
+ "for survivors:", r->bottom(), original_top, r->end());
+ r->print();
+ }
+ g1_policy()->record_before_bytes(r_used);
+ }
+}
+
+void G1CollectedHeap::push_gc_alloc_region(HeapRegion* hr) {
+ assert(Thread::current()->is_VM_thread() ||
+ par_alloc_during_gc_lock()->owned_by_self(), "Precondition");
+ assert(!hr->is_gc_alloc_region() && !hr->in_collection_set(),
+ "Precondition.");
+ hr->set_is_gc_alloc_region(true);
+ hr->set_next_gc_alloc_region(_gc_alloc_region_list);
+ _gc_alloc_region_list = hr;
+}
+
+#ifdef G1_DEBUG
+class FindGCAllocRegion: public HeapRegionClosure {
+public:
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->is_gc_alloc_region()) {
+ gclog_or_tty->print_cr("Region %d ["PTR_FORMAT"...] is still a gc_alloc_region.",
+ r->hrs_index(), r->bottom());
+ }
+ return false;
+ }
+};
+#endif // G1_DEBUG
+
+void G1CollectedHeap::forget_alloc_region_list() {
+ assert(Thread::current()->is_VM_thread(), "Precondition");
+ while (_gc_alloc_region_list != NULL) {
+ HeapRegion* r = _gc_alloc_region_list;
+ assert(r->is_gc_alloc_region(), "Invariant.");
+ _gc_alloc_region_list = r->next_gc_alloc_region();
+ r->set_next_gc_alloc_region(NULL);
+ r->set_is_gc_alloc_region(false);
+ if (r->is_empty()) {
+ ++_free_regions;
+ }
+ }
+#ifdef G1_DEBUG
+ FindGCAllocRegion fa;
+ heap_region_iterate(&fa);
+#endif // G1_DEBUG
+}
+
+
+bool G1CollectedHeap::check_gc_alloc_regions() {
+ // TODO: allocation regions check
+ return true;
+}
+
+void G1CollectedHeap::get_gc_alloc_regions() {
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ // Create new GC alloc regions.
+ HeapRegion* alloc_region = _gc_alloc_regions[ap];
+ // Clear this alloc region, so that in case it turns out to be
+ // unacceptable, we end up with no allocation region, rather than a bad
+ // one.
+ _gc_alloc_regions[ap] = NULL;
+ if (alloc_region == NULL || alloc_region->in_collection_set()) {
+ // Can't re-use old one. Allocate a new one.
+ alloc_region = newAllocRegionWithExpansion(ap, 0);
+ }
+ if (alloc_region != NULL) {
+ set_gc_alloc_region(ap, alloc_region);
+ }
+ }
+ // Set alternative regions for allocation purposes that have reached
+ // thier limit.
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ GCAllocPurpose alt_purpose = g1_policy()->alternative_purpose(ap);
+ if (_gc_alloc_regions[ap] == NULL && alt_purpose != ap) {
+ _gc_alloc_regions[ap] = _gc_alloc_regions[alt_purpose];
+ }
+ }
+ assert(check_gc_alloc_regions(), "alloc regions messed up");
+}
+
+void G1CollectedHeap::release_gc_alloc_regions() {
+ // We keep a separate list of all regions that have been alloc regions in
+ // the current collection pause. Forget that now.
+ forget_alloc_region_list();
+
+ // The current alloc regions contain objs that have survived
+ // collection. Make them no longer GC alloc regions.
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ HeapRegion* r = _gc_alloc_regions[ap];
+ if (r != NULL && r->is_empty()) {
+ {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ r->set_zero_fill_complete();
+ put_free_region_on_list_locked(r);
+ }
+ }
+ // set_gc_alloc_region will also NULLify all aliases to the region
+ set_gc_alloc_region(ap, NULL);
+ _gc_alloc_region_counts[ap] = 0;
+ }
+}
+
+void G1CollectedHeap::init_for_evac_failure(OopsInHeapRegionClosure* cl) {
+ _drain_in_progress = false;
+ set_evac_failure_closure(cl);
+ _evac_failure_scan_stack = new (ResourceObj::C_HEAP) GrowableArray(40, true);
+}
+
+void G1CollectedHeap::finalize_for_evac_failure() {
+ assert(_evac_failure_scan_stack != NULL &&
+ _evac_failure_scan_stack->length() == 0,
+ "Postcondition");
+ assert(!_drain_in_progress, "Postcondition");
+ // Don't have to delete, since the scan stack is a resource object.
+ _evac_failure_scan_stack = NULL;
+}
+
+
+
+// *** Sequential G1 Evacuation
+
+HeapWord* G1CollectedHeap::allocate_during_gc(GCAllocPurpose purpose, size_t word_size) {
+ HeapRegion* alloc_region = _gc_alloc_regions[purpose];
+ // let the caller handle alloc failure
+ if (alloc_region == NULL) return NULL;
+ assert(isHumongous(word_size) || !alloc_region->isHumongous(),
+ "Either the object is humongous or the region isn't");
+ HeapWord* block = alloc_region->allocate(word_size);
+ if (block == NULL) {
+ block = allocate_during_gc_slow(purpose, alloc_region, false, word_size);
+ }
+ return block;
+}
+
+class G1IsAliveClosure: public BoolObjectClosure {
+ G1CollectedHeap* _g1;
+public:
+ G1IsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
+ void do_object(oop p) { assert(false, "Do not call."); }
+ bool do_object_b(oop p) {
+ // It is reachable if it is outside the collection set, or is inside
+ // and forwarded.
+
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr("is alive "PTR_FORMAT" in CS %d forwarded %d overall %d",
+ (void*) p, _g1->obj_in_cs(p), p->is_forwarded(),
+ !_g1->obj_in_cs(p) || p->is_forwarded());
+#endif // G1_DEBUG
+
+ return !_g1->obj_in_cs(p) || p->is_forwarded();
+ }
+};
+
+class G1KeepAliveClosure: public OopClosure {
+ G1CollectedHeap* _g1;
+public:
+ G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+ void do_oop(oop* p) {
+ oop obj = *p;
+#ifdef G1_DEBUG
+ if (PrintGC && Verbose) {
+ gclog_or_tty->print_cr("keep alive *"PTR_FORMAT" = "PTR_FORMAT" "PTR_FORMAT,
+ p, (void*) obj, (void*) *p);
+ }
+#endif // G1_DEBUG
+
+ if (_g1->obj_in_cs(obj)) {
+ assert( obj->is_forwarded(), "invariant" );
+ *p = obj->forwardee();
+
+#ifdef G1_DEBUG
+ gclog_or_tty->print_cr(" in CSet: moved "PTR_FORMAT" -> "PTR_FORMAT,
+ (void*) obj, (void*) *p);
+#endif // G1_DEBUG
+ }
+ }
+};
+
+class RecreateRSetEntriesClosure: public OopClosure {
+private:
+ G1CollectedHeap* _g1;
+ G1RemSet* _g1_rem_set;
+ HeapRegion* _from;
+public:
+ RecreateRSetEntriesClosure(G1CollectedHeap* g1, HeapRegion* from) :
+ _g1(g1), _g1_rem_set(g1->g1_rem_set()), _from(from)
+ {}
+
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+ void do_oop(oop* p) {
+ assert(_from->is_in_reserved(p), "paranoia");
+ if (*p != NULL) {
+ _g1_rem_set->write_ref(_from, p);
+ }
+ }
+};
+
+class RemoveSelfPointerClosure: public ObjectClosure {
+private:
+ G1CollectedHeap* _g1;
+ ConcurrentMark* _cm;
+ HeapRegion* _hr;
+ size_t _prev_marked_bytes;
+ size_t _next_marked_bytes;
+public:
+ RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr) :
+ _g1(g1), _cm(_g1->concurrent_mark()), _hr(hr),
+ _prev_marked_bytes(0), _next_marked_bytes(0)
+ {}
+
+ size_t prev_marked_bytes() { return _prev_marked_bytes; }
+ size_t next_marked_bytes() { return _next_marked_bytes; }
+
+ // The original idea here was to coalesce evacuated and dead objects.
+ // However that caused complications with the block offset table (BOT).
+ // In particular if there were two TLABs, one of them partially refined.
+ // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~|
+ // The BOT entries of the unrefined part of TLAB_2 point to the start
+ // of TLAB_2. If the last object of the TLAB_1 and the first object
+ // of TLAB_2 are coalesced, then the cards of the unrefined part
+ // would point into middle of the filler object.
+ //
+ // The current approach is to not coalesce and leave the BOT contents intact.
+ void do_object(oop obj) {
+ if (obj->is_forwarded() && obj->forwardee() == obj) {
+ // The object failed to move.
+ assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs.");
+ _cm->markPrev(obj);
+ assert(_cm->isPrevMarked(obj), "Should be marked!");
+ _prev_marked_bytes += (obj->size() * HeapWordSize);
+ if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) {
+ _cm->markAndGrayObjectIfNecessary(obj);
+ }
+ obj->set_mark(markOopDesc::prototype());
+ // While we were processing RSet buffers during the
+ // collection, we actually didn't scan any cards on the
+ // collection set, since we didn't want to update remebered
+ // sets with entries that point into the collection set, given
+ // that live objects fromthe collection set are about to move
+ // and such entries will be stale very soon. This change also
+ // dealt with a reliability issue which involved scanning a
+ // card in the collection set and coming across an array that
+ // was being chunked and looking malformed. The problem is
+ // that, if evacuation fails, we might have remembered set
+ // entries missing given that we skipped cards on the
+ // collection set. So, we'll recreate such entries now.
+ RecreateRSetEntriesClosure cl(_g1, _hr);
+ obj->oop_iterate(&cl);
+ assert(_cm->isPrevMarked(obj), "Should be marked!");
+ } else {
+ // The object has been either evacuated or is dead. Fill it with a
+ // dummy object.
+ MemRegion mr((HeapWord*)obj, obj->size());
+ CollectedHeap::fill_with_object(mr);
+ _cm->clearRangeBothMaps(mr);
+ }
+ }
+};
+
+void G1CollectedHeap::remove_self_forwarding_pointers() {
+ HeapRegion* cur = g1_policy()->collection_set();
+
+ while (cur != NULL) {
+ assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
+
+ if (cur->evacuation_failed()) {
+ RemoveSelfPointerClosure rspc(_g1h, cur);
+ assert(cur->in_collection_set(), "bad CS");
+ cur->object_iterate(&rspc);
+
+ // A number of manipulations to make the TAMS be the current top,
+ // and the marked bytes be the ones observed in the iteration.
+ if (_g1h->concurrent_mark()->at_least_one_mark_complete()) {
+ // The comments below are the postconditions achieved by the
+ // calls. Note especially the last such condition, which says that
+ // the count of marked bytes has been properly restored.
+ cur->note_start_of_marking(false);
+ // _next_top_at_mark_start == top, _next_marked_bytes == 0
+ cur->add_to_marked_bytes(rspc.prev_marked_bytes());
+ // _next_marked_bytes == prev_marked_bytes.
+ cur->note_end_of_marking();
+ // _prev_top_at_mark_start == top(),
+ // _prev_marked_bytes == prev_marked_bytes
+ }
+ // If there is no mark in progress, we modified the _next variables
+ // above needlessly, but harmlessly.
+ if (_g1h->mark_in_progress()) {
+ cur->note_start_of_marking(false);
+ // _next_top_at_mark_start == top, _next_marked_bytes == 0
+ // _next_marked_bytes == next_marked_bytes.
+ }
+
+ // Now make sure the region has the right index in the sorted array.
+ g1_policy()->note_change_in_marked_bytes(cur);
+ }
+ cur = cur->next_in_collection_set();
+ }
+ assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!");
+
+ // Now restore saved marks, if any.
+ if (_objs_with_preserved_marks != NULL) {
+ assert(_preserved_marks_of_objs != NULL, "Both or none.");
+ assert(_objs_with_preserved_marks->length() ==
+ _preserved_marks_of_objs->length(), "Both or none.");
+ guarantee(_objs_with_preserved_marks->length() ==
+ _preserved_marks_of_objs->length(), "Both or none.");
+ for (int i = 0; i < _objs_with_preserved_marks->length(); i++) {
+ oop obj = _objs_with_preserved_marks->at(i);
+ markOop m = _preserved_marks_of_objs->at(i);
+ obj->set_mark(m);
+ }
+ // Delete the preserved marks growable arrays (allocated on the C heap).
+ delete _objs_with_preserved_marks;
+ delete _preserved_marks_of_objs;
+ _objs_with_preserved_marks = NULL;
+ _preserved_marks_of_objs = NULL;
+ }
+}
+
+void G1CollectedHeap::push_on_evac_failure_scan_stack(oop obj) {
+ _evac_failure_scan_stack->push(obj);
+}
+
+void G1CollectedHeap::drain_evac_failure_scan_stack() {
+ assert(_evac_failure_scan_stack != NULL, "precondition");
+
+ while (_evac_failure_scan_stack->length() > 0) {
+ oop obj = _evac_failure_scan_stack->pop();
+ _evac_failure_closure->set_region(heap_region_containing(obj));
+ obj->oop_iterate_backwards(_evac_failure_closure);
+ }
+}
+
+void G1CollectedHeap::handle_evacuation_failure(oop old) {
+ markOop m = old->mark();
+ // forward to self
+ assert(!old->is_forwarded(), "precondition");
+
+ old->forward_to(old);
+ handle_evacuation_failure_common(old, m);
+}
+
+oop
+G1CollectedHeap::handle_evacuation_failure_par(OopsInHeapRegionClosure* cl,
+ oop old) {
+ markOop m = old->mark();
+ oop forward_ptr = old->forward_to_atomic(old);
+ if (forward_ptr == NULL) {
+ // Forward-to-self succeeded.
+ if (_evac_failure_closure != cl) {
+ MutexLockerEx x(EvacFailureStack_lock, Mutex::_no_safepoint_check_flag);
+ assert(!_drain_in_progress,
+ "Should only be true while someone holds the lock.");
+ // Set the global evac-failure closure to the current thread's.
+ assert(_evac_failure_closure == NULL, "Or locking has failed.");
+ set_evac_failure_closure(cl);
+ // Now do the common part.
+ handle_evacuation_failure_common(old, m);
+ // Reset to NULL.
+ set_evac_failure_closure(NULL);
+ } else {
+ // The lock is already held, and this is recursive.
+ assert(_drain_in_progress, "This should only be the recursive case.");
+ handle_evacuation_failure_common(old, m);
+ }
+ return old;
+ } else {
+ // Someone else had a place to copy it.
+ return forward_ptr;
+ }
+}
+
+void G1CollectedHeap::handle_evacuation_failure_common(oop old, markOop m) {
+ set_evacuation_failed(true);
+
+ preserve_mark_if_necessary(old, m);
+
+ HeapRegion* r = heap_region_containing(old);
+ if (!r->evacuation_failed()) {
+ r->set_evacuation_failed(true);
+ if (G1TraceRegions) {
+ gclog_or_tty->print("evacuation failed in heap region "PTR_FORMAT" "
+ "["PTR_FORMAT","PTR_FORMAT")\n",
+ r, r->bottom(), r->end());
+ }
+ }
+
+ push_on_evac_failure_scan_stack(old);
+
+ if (!_drain_in_progress) {
+ // prevent recursion in copy_to_survivor_space()
+ _drain_in_progress = true;
+ drain_evac_failure_scan_stack();
+ _drain_in_progress = false;
+ }
+}
+
+void G1CollectedHeap::preserve_mark_if_necessary(oop obj, markOop m) {
+ if (m != markOopDesc::prototype()) {
+ if (_objs_with_preserved_marks == NULL) {
+ assert(_preserved_marks_of_objs == NULL, "Both or none.");
+ _objs_with_preserved_marks =
+ new (ResourceObj::C_HEAP) GrowableArray(40, true);
+ _preserved_marks_of_objs =
+ new (ResourceObj::C_HEAP) GrowableArray(40, true);
+ }
+ _objs_with_preserved_marks->push(obj);
+ _preserved_marks_of_objs->push(m);
+ }
+}
+
+// *** Parallel G1 Evacuation
+
+HeapWord* G1CollectedHeap::par_allocate_during_gc(GCAllocPurpose purpose,
+ size_t word_size) {
+ HeapRegion* alloc_region = _gc_alloc_regions[purpose];
+ // let the caller handle alloc failure
+ if (alloc_region == NULL) return NULL;
+
+ HeapWord* block = alloc_region->par_allocate(word_size);
+ if (block == NULL) {
+ MutexLockerEx x(par_alloc_during_gc_lock(),
+ Mutex::_no_safepoint_check_flag);
+ block = allocate_during_gc_slow(purpose, alloc_region, true, word_size);
+ }
+ return block;
+}
+
+HeapWord*
+G1CollectedHeap::allocate_during_gc_slow(GCAllocPurpose purpose,
+ HeapRegion* alloc_region,
+ bool par,
+ size_t word_size) {
+ HeapWord* block = NULL;
+ // In the parallel case, a previous thread to obtain the lock may have
+ // already assigned a new gc_alloc_region.
+ if (alloc_region != _gc_alloc_regions[purpose]) {
+ assert(par, "But should only happen in parallel case.");
+ alloc_region = _gc_alloc_regions[purpose];
+ if (alloc_region == NULL) return NULL;
+ block = alloc_region->par_allocate(word_size);
+ if (block != NULL) return block;
+ // Otherwise, continue; this new region is empty, too.
+ }
+ assert(alloc_region != NULL, "We better have an allocation region");
+ // Another thread might have obtained alloc_region for the given
+ // purpose, and might be attempting to allocate in it, and might
+ // succeed. Therefore, we can't do the "finalization" stuff on the
+ // region below until we're sure the last allocation has happened.
+ // We ensure this by allocating the remaining space with a garbage
+ // object.
+ if (par) par_allocate_remaining_space(alloc_region);
+ // Now we can do the post-GC stuff on the region.
+ alloc_region->note_end_of_copying();
+ g1_policy()->record_after_bytes(alloc_region->used());
+
+ if (_gc_alloc_region_counts[purpose] >= g1_policy()->max_regions(purpose)) {
+ // Cannot allocate more regions for the given purpose.
+ GCAllocPurpose alt_purpose = g1_policy()->alternative_purpose(purpose);
+ // Is there an alternative?
+ if (purpose != alt_purpose) {
+ HeapRegion* alt_region = _gc_alloc_regions[alt_purpose];
+ // Has not the alternative region been aliased?
+ if (alloc_region != alt_region) {
+ // Try to allocate in the alternative region.
+ if (par) {
+ block = alt_region->par_allocate(word_size);
+ } else {
+ block = alt_region->allocate(word_size);
+ }
+ // Make an alias.
+ _gc_alloc_regions[purpose] = _gc_alloc_regions[alt_purpose];
+ }
+ if (block != NULL) {
+ return block;
+ }
+ // Both the allocation region and the alternative one are full
+ // and aliased, replace them with a new allocation region.
+ purpose = alt_purpose;
+ } else {
+ set_gc_alloc_region(purpose, NULL);
+ return NULL;
+ }
+ }
+
+ // Now allocate a new region for allocation.
+ alloc_region = newAllocRegionWithExpansion(purpose, word_size, false /*zero_filled*/);
+
+ // let the caller handle alloc failure
+ if (alloc_region != NULL) {
+
+ assert(check_gc_alloc_regions(), "alloc regions messed up");
+ assert(alloc_region->saved_mark_at_top(),
+ "Mark should have been saved already.");
+ // We used to assert that the region was zero-filled here, but no
+ // longer.
+
+ // This must be done last: once it's installed, other regions may
+ // allocate in it (without holding the lock.)
+ set_gc_alloc_region(purpose, alloc_region);
+
+ if (par) {
+ block = alloc_region->par_allocate(word_size);
+ } else {
+ block = alloc_region->allocate(word_size);
+ }
+ // Caller handles alloc failure.
+ } else {
+ // This sets other apis using the same old alloc region to NULL, also.
+ set_gc_alloc_region(purpose, NULL);
+ }
+ return block; // May be NULL.
+}
+
+void G1CollectedHeap::par_allocate_remaining_space(HeapRegion* r) {
+ HeapWord* block = NULL;
+ size_t free_words;
+ do {
+ free_words = r->free()/HeapWordSize;
+ // If there's too little space, no one can allocate, so we're done.
+ if (free_words < (size_t)oopDesc::header_size()) return;
+ // Otherwise, try to claim it.
+ block = r->par_allocate(free_words);
+ } while (block == NULL);
+ fill_with_object(block, free_words);
+}
+
+#define use_local_bitmaps 1
+#define verify_local_bitmaps 0
+
+#ifndef PRODUCT
+
+class GCLabBitMap;
+class GCLabBitMapClosure: public BitMapClosure {
+private:
+ ConcurrentMark* _cm;
+ GCLabBitMap* _bitmap;
+
+public:
+ GCLabBitMapClosure(ConcurrentMark* cm,
+ GCLabBitMap* bitmap) {
+ _cm = cm;
+ _bitmap = bitmap;
+ }
+
+ virtual bool do_bit(size_t offset);
+};
+
+#endif // PRODUCT
+
+#define oop_buffer_length 256
+
+class GCLabBitMap: public BitMap {
+private:
+ ConcurrentMark* _cm;
+
+ int _shifter;
+ size_t _bitmap_word_covers_words;
+
+ // beginning of the heap
+ HeapWord* _heap_start;
+
+ // this is the actual start of the GCLab
+ HeapWord* _real_start_word;
+
+ // this is the actual end of the GCLab
+ HeapWord* _real_end_word;
+
+ // this is the first word, possibly located before the actual start
+ // of the GCLab, that corresponds to the first bit of the bitmap
+ HeapWord* _start_word;
+
+ // size of a GCLab in words
+ size_t _gclab_word_size;
+
+ static int shifter() {
+ return MinObjAlignment - 1;
+ }
+
+ // how many heap words does a single bitmap word corresponds to?
+ static size_t bitmap_word_covers_words() {
+ return BitsPerWord << shifter();
+ }
+
+ static size_t gclab_word_size() {
+ return ParallelGCG1AllocBufferSize / HeapWordSize;
+ }
+
+ static size_t bitmap_size_in_bits() {
+ size_t bits_in_bitmap = gclab_word_size() >> shifter();
+ // We are going to ensure that the beginning of a word in this
+ // bitmap also corresponds to the beginning of a word in the
+ // global marking bitmap. To handle the case where a GCLab
+ // starts from the middle of the bitmap, we need to add enough
+ // space (i.e. up to a bitmap word) to ensure that we have
+ // enough bits in the bitmap.
+ return bits_in_bitmap + BitsPerWord - 1;
+ }
+public:
+ GCLabBitMap(HeapWord* heap_start)
+ : BitMap(bitmap_size_in_bits()),
+ _cm(G1CollectedHeap::heap()->concurrent_mark()),
+ _shifter(shifter()),
+ _bitmap_word_covers_words(bitmap_word_covers_words()),
+ _heap_start(heap_start),
+ _gclab_word_size(gclab_word_size()),
+ _real_start_word(NULL),
+ _real_end_word(NULL),
+ _start_word(NULL)
+ {
+ guarantee( size_in_words() >= bitmap_size_in_words(),
+ "just making sure");
+ }
+
+ inline unsigned heapWordToOffset(HeapWord* addr) {
+ unsigned offset = (unsigned) pointer_delta(addr, _start_word) >> _shifter;
+ assert(offset < size(), "offset should be within bounds");
+ return offset;
+ }
+
+ inline HeapWord* offsetToHeapWord(size_t offset) {
+ HeapWord* addr = _start_word + (offset << _shifter);
+ assert(_real_start_word <= addr && addr < _real_end_word, "invariant");
+ return addr;
+ }
+
+ bool fields_well_formed() {
+ bool ret1 = (_real_start_word == NULL) &&
+ (_real_end_word == NULL) &&
+ (_start_word == NULL);
+ if (ret1)
+ return true;
+
+ bool ret2 = _real_start_word >= _start_word &&
+ _start_word < _real_end_word &&
+ (_real_start_word + _gclab_word_size) == _real_end_word &&
+ (_start_word + _gclab_word_size + _bitmap_word_covers_words)
+ > _real_end_word;
+ return ret2;
+ }
+
+ inline bool mark(HeapWord* addr) {
+ guarantee(use_local_bitmaps, "invariant");
+ assert(fields_well_formed(), "invariant");
+
+ if (addr >= _real_start_word && addr < _real_end_word) {
+ assert(!isMarked(addr), "should not have already been marked");
+
+ // first mark it on the bitmap
+ at_put(heapWordToOffset(addr), true);
+
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ inline bool isMarked(HeapWord* addr) {
+ guarantee(use_local_bitmaps, "invariant");
+ assert(fields_well_formed(), "invariant");
+
+ return at(heapWordToOffset(addr));
+ }
+
+ void set_buffer(HeapWord* start) {
+ guarantee(use_local_bitmaps, "invariant");
+ clear();
+
+ assert(start != NULL, "invariant");
+ _real_start_word = start;
+ _real_end_word = start + _gclab_word_size;
+
+ size_t diff =
+ pointer_delta(start, _heap_start) % _bitmap_word_covers_words;
+ _start_word = start - diff;
+
+ assert(fields_well_formed(), "invariant");
+ }
+
+#ifndef PRODUCT
+ void verify() {
+ // verify that the marks have been propagated
+ GCLabBitMapClosure cl(_cm, this);
+ iterate(&cl);
+ }
+#endif // PRODUCT
+
+ void retire() {
+ guarantee(use_local_bitmaps, "invariant");
+ assert(fields_well_formed(), "invariant");
+
+ if (_start_word != NULL) {
+ CMBitMap* mark_bitmap = _cm->nextMarkBitMap();
+
+ // this means that the bitmap was set up for the GCLab
+ assert(_real_start_word != NULL && _real_end_word != NULL, "invariant");
+
+ mark_bitmap->mostly_disjoint_range_union(this,
+ 0, // always start from the start of the bitmap
+ _start_word,
+ size_in_words());
+ _cm->grayRegionIfNecessary(MemRegion(_real_start_word, _real_end_word));
+
+#ifndef PRODUCT
+ if (use_local_bitmaps && verify_local_bitmaps)
+ verify();
+#endif // PRODUCT
+ } else {
+ assert(_real_start_word == NULL && _real_end_word == NULL, "invariant");
+ }
+ }
+
+ static size_t bitmap_size_in_words() {
+ return (bitmap_size_in_bits() + BitsPerWord - 1) / BitsPerWord;
+ }
+};
+
+#ifndef PRODUCT
+
+bool GCLabBitMapClosure::do_bit(size_t offset) {
+ HeapWord* addr = _bitmap->offsetToHeapWord(offset);
+ guarantee(_cm->isMarked(oop(addr)), "it should be!");
+ return true;
+}
+
+#endif // PRODUCT
+
+class G1ParGCAllocBuffer: public ParGCAllocBuffer {
+private:
+ bool _retired;
+ bool _during_marking;
+ GCLabBitMap _bitmap;
+
+public:
+ G1ParGCAllocBuffer() :
+ ParGCAllocBuffer(ParallelGCG1AllocBufferSize / HeapWordSize),
+ _during_marking(G1CollectedHeap::heap()->mark_in_progress()),
+ _bitmap(G1CollectedHeap::heap()->reserved_region().start()),
+ _retired(false)
+ { }
+
+ inline bool mark(HeapWord* addr) {
+ guarantee(use_local_bitmaps, "invariant");
+ assert(_during_marking, "invariant");
+ return _bitmap.mark(addr);
+ }
+
+ inline void set_buf(HeapWord* buf) {
+ if (use_local_bitmaps && _during_marking)
+ _bitmap.set_buffer(buf);
+ ParGCAllocBuffer::set_buf(buf);
+ _retired = false;
+ }
+
+ inline void retire(bool end_of_gc, bool retain) {
+ if (_retired)
+ return;
+ if (use_local_bitmaps && _during_marking) {
+ _bitmap.retire();
+ }
+ ParGCAllocBuffer::retire(end_of_gc, retain);
+ _retired = true;
+ }
+};
+
+
+class G1ParScanThreadState : public StackObj {
+protected:
+ G1CollectedHeap* _g1h;
+ RefToScanQueue* _refs;
+
+ typedef GrowableArray OverflowQueue;
+ OverflowQueue* _overflowed_refs;
+
+ G1ParGCAllocBuffer _alloc_buffers[GCAllocPurposeCount];
+
+ size_t _alloc_buffer_waste;
+ size_t _undo_waste;
+
+ OopsInHeapRegionClosure* _evac_failure_cl;
+ G1ParScanHeapEvacClosure* _evac_cl;
+ G1ParScanPartialArrayClosure* _partial_scan_cl;
+
+ int _hash_seed;
+ int _queue_num;
+
+ int _term_attempts;
+#if G1_DETAILED_STATS
+ int _pushes, _pops, _steals, _steal_attempts;
+ int _overflow_pushes;
+#endif
+
+ double _start;
+ double _start_strong_roots;
+ double _strong_roots_time;
+ double _start_term;
+ double _term_time;
+
+ // Map from young-age-index (0 == not young, 1 is youngest) to
+ // surviving words. base is what we get back from the malloc call
+ size_t* _surviving_young_words_base;
+ // this points into the array, as we use the first few entries for padding
+ size_t* _surviving_young_words;
+
+#define PADDING_ELEM_NUM (64 / sizeof(size_t))
+
+ void add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; }
+
+ void add_to_undo_waste(size_t waste) { _undo_waste += waste; }
+
+public:
+ G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num)
+ : _g1h(g1h),
+ _refs(g1h->task_queue(queue_num)),
+ _hash_seed(17), _queue_num(queue_num),
+ _term_attempts(0),
+#if G1_DETAILED_STATS
+ _pushes(0), _pops(0), _steals(0),
+ _steal_attempts(0), _overflow_pushes(0),
+#endif
+ _strong_roots_time(0), _term_time(0),
+ _alloc_buffer_waste(0), _undo_waste(0)
+ {
+ // we allocate G1YoungSurvRateNumRegions plus one entries, since
+ // we "sacrifice" entry 0 to keep track of surviving bytes for
+ // non-young regions (where the age is -1)
+ // We also add a few elements at the beginning and at the end in
+ // an attempt to eliminate cache contention
+ size_t real_length = 1 + _g1h->g1_policy()->young_cset_length();
+ size_t array_length = PADDING_ELEM_NUM +
+ real_length +
+ PADDING_ELEM_NUM;
+ _surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length);
+ if (_surviving_young_words_base == NULL)
+ vm_exit_out_of_memory(array_length * sizeof(size_t),
+ "Not enough space for young surv histo.");
+ _surviving_young_words = _surviving_young_words_base + PADDING_ELEM_NUM;
+ memset(_surviving_young_words, 0, real_length * sizeof(size_t));
+
+ _overflowed_refs = new OverflowQueue(10);
+
+ _start = os::elapsedTime();
+ }
+
+ ~G1ParScanThreadState() {
+ FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_base);
+ }
+
+ RefToScanQueue* refs() { return _refs; }
+ OverflowQueue* overflowed_refs() { return _overflowed_refs; }
+
+ inline G1ParGCAllocBuffer* alloc_buffer(GCAllocPurpose purpose) {
+ return &_alloc_buffers[purpose];
+ }
+
+ size_t alloc_buffer_waste() { return _alloc_buffer_waste; }
+ size_t undo_waste() { return _undo_waste; }
+
+ void push_on_queue(oop* ref) {
+ if (!refs()->push(ref)) {
+ overflowed_refs()->push(ref);
+ IF_G1_DETAILED_STATS(note_overflow_push());
+ } else {
+ IF_G1_DETAILED_STATS(note_push());
+ }
+ }
+
+ void pop_from_queue(oop*& ref) {
+ if (!refs()->pop_local(ref)) {
+ ref = NULL;
+ } else {
+ IF_G1_DETAILED_STATS(note_pop());
+ }
+ }
+
+ void pop_from_overflow_queue(oop*& ref) {
+ ref = overflowed_refs()->pop();
+ }
+
+ int refs_to_scan() { return refs()->size(); }
+ int overflowed_refs_to_scan() { return overflowed_refs()->length(); }
+
+ HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) {
+
+ HeapWord* obj = NULL;
+ if (word_sz * 100 <
+ (size_t)(ParallelGCG1AllocBufferSize / HeapWordSize) *
+ ParallelGCBufferWastePct) {
+ G1ParGCAllocBuffer* alloc_buf = alloc_buffer(purpose);
+ add_to_alloc_buffer_waste(alloc_buf->words_remaining());
+ alloc_buf->retire(false, false);
+
+ HeapWord* buf =
+ _g1h->par_allocate_during_gc(purpose, ParallelGCG1AllocBufferSize / HeapWordSize);
+ if (buf == NULL) return NULL; // Let caller handle allocation failure.
+ // Otherwise.
+ alloc_buf->set_buf(buf);
+
+ obj = alloc_buf->allocate(word_sz);
+ assert(obj != NULL, "buffer was definitely big enough...");
+ }
+ else {
+ obj = _g1h->par_allocate_during_gc(purpose, word_sz);
+ }
+ return obj;
+ }
+
+ HeapWord* allocate(GCAllocPurpose purpose, size_t word_sz) {
+ HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
+ if (obj != NULL) return obj;
+ return allocate_slow(purpose, word_sz);
+ }
+
+ void undo_allocation(GCAllocPurpose purpose, HeapWord* obj, size_t word_sz) {
+ if (alloc_buffer(purpose)->contains(obj)) {
+ guarantee(alloc_buffer(purpose)->contains(obj + word_sz - 1),
+ "should contain whole object");
+ alloc_buffer(purpose)->undo_allocation(obj, word_sz);
+ } else {
+ CollectedHeap::fill_with_object(obj, word_sz);
+ add_to_undo_waste(word_sz);
+ }
+ }
+
+ void set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_cl) {
+ _evac_failure_cl = evac_failure_cl;
+ }
+ OopsInHeapRegionClosure* evac_failure_closure() {
+ return _evac_failure_cl;
+ }
+
+ void set_evac_closure(G1ParScanHeapEvacClosure* evac_cl) {
+ _evac_cl = evac_cl;
+ }
+
+ void set_partial_scan_closure(G1ParScanPartialArrayClosure* partial_scan_cl) {
+ _partial_scan_cl = partial_scan_cl;
+ }
+
+ int* hash_seed() { return &_hash_seed; }
+ int queue_num() { return _queue_num; }
+
+ int term_attempts() { return _term_attempts; }
+ void note_term_attempt() { _term_attempts++; }
+
+#if G1_DETAILED_STATS
+ int pushes() { return _pushes; }
+ int pops() { return _pops; }
+ int steals() { return _steals; }
+ int steal_attempts() { return _steal_attempts; }
+ int overflow_pushes() { return _overflow_pushes; }
+
+ void note_push() { _pushes++; }
+ void note_pop() { _pops++; }
+ void note_steal() { _steals++; }
+ void note_steal_attempt() { _steal_attempts++; }
+ void note_overflow_push() { _overflow_pushes++; }
+#endif
+
+ void start_strong_roots() {
+ _start_strong_roots = os::elapsedTime();
+ }
+ void end_strong_roots() {
+ _strong_roots_time += (os::elapsedTime() - _start_strong_roots);
+ }
+ double strong_roots_time() { return _strong_roots_time; }
+
+ void start_term_time() {
+ note_term_attempt();
+ _start_term = os::elapsedTime();
+ }
+ void end_term_time() {
+ _term_time += (os::elapsedTime() - _start_term);
+ }
+ double term_time() { return _term_time; }
+
+ double elapsed() {
+ return os::elapsedTime() - _start;
+ }
+
+ size_t* surviving_young_words() {
+ // We add on to hide entry 0 which accumulates surviving words for
+ // age -1 regions (i.e. non-young ones)
+ return _surviving_young_words;
+ }
+
+ void retire_alloc_buffers() {
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ size_t waste = _alloc_buffers[ap].words_remaining();
+ add_to_alloc_buffer_waste(waste);
+ _alloc_buffers[ap].retire(true, false);
+ }
+ }
+
+ void trim_queue() {
+ while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
+ oop *ref_to_scan = NULL;
+ if (overflowed_refs_to_scan() == 0) {
+ pop_from_queue(ref_to_scan);
+ } else {
+ pop_from_overflow_queue(ref_to_scan);
+ }
+ if (ref_to_scan != NULL) {
+ if ((intptr_t)ref_to_scan & G1_PARTIAL_ARRAY_MASK) {
+ _partial_scan_cl->do_oop_nv(ref_to_scan);
+ } else {
+ // Note: we can use "raw" versions of "region_containing" because
+ // "obj_to_scan" is definitely in the heap, and is not in a
+ // humongous region.
+ HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
+ _evac_cl->set_region(r);
+ _evac_cl->do_oop_nv(ref_to_scan);
+ }
+ }
+ }
+ }
+};
+
+
+G1ParClosureSuper::G1ParClosureSuper(G1CollectedHeap* g1, G1ParScanThreadState* par_scan_state) :
+ _g1(g1), _g1_rem(_g1->g1_rem_set()), _cm(_g1->concurrent_mark()),
+ _par_scan_state(par_scan_state) { }
+
+// This closure is applied to the fields of the objects that have just been copied.
+// Should probably be made inline and moved in g1OopClosures.inline.hpp.
+void G1ParScanClosure::do_oop_nv(oop* p) {
+ oop obj = *p;
+ if (obj != NULL) {
+ if (_g1->obj_in_cs(obj)) {
+ if (obj->is_forwarded()) {
+ *p = obj->forwardee();
+ } else {
+ _par_scan_state->push_on_queue(p);
+ return;
+ }
+ }
+ _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+ }
+}
+
+void G1ParCopyHelper::mark_forwardee(oop* p) {
+ // This is called _after_ do_oop_work has been called, hence after
+ // the object has been relocated to its new location and *p points
+ // to its new location.
+
+ oop thisOop = *p;
+ if (thisOop != NULL) {
+ assert((_g1->evacuation_failed()) || (!_g1->obj_in_cs(thisOop)),
+ "shouldn't still be in the CSet if evacuation didn't fail.");
+ HeapWord* addr = (HeapWord*)thisOop;
+ if (_g1->is_in_g1_reserved(addr))
+ _cm->grayRoot(oop(addr));
+ }
+}
+
+oop G1ParCopyHelper::copy_to_survivor_space(oop old) {
+ size_t word_sz = old->size();
+ HeapRegion* from_region = _g1->heap_region_containing_raw(old);
+ // +1 to make the -1 indexes valid...
+ int young_index = from_region->young_index_in_cset()+1;
+ assert( (from_region->is_young() && young_index > 0) ||
+ (!from_region->is_young() && young_index == 0), "invariant" );
+ G1CollectorPolicy* g1p = _g1->g1_policy();
+ markOop m = old->mark();
+ GCAllocPurpose alloc_purpose = g1p->evacuation_destination(from_region, m->age(),
+ word_sz);
+ HeapWord* obj_ptr = _par_scan_state->allocate(alloc_purpose, word_sz);
+ oop obj = oop(obj_ptr);
+
+ if (obj_ptr == NULL) {
+ // This will either forward-to-self, or detect that someone else has
+ // installed a forwarding pointer.
+ OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
+ return _g1->handle_evacuation_failure_par(cl, old);
+ }
+
+ oop forward_ptr = old->forward_to_atomic(obj);
+ if (forward_ptr == NULL) {
+ Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
+ obj->set_mark(m);
+ if (g1p->track_object_age(alloc_purpose)) {
+ obj->incr_age();
+ }
+ // preserve "next" mark bit
+ if (_g1->mark_in_progress() && !_g1->is_obj_ill(old)) {
+ if (!use_local_bitmaps ||
+ !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) {
+ // if we couldn't mark it on the local bitmap (this happens when
+ // the object was not allocated in the GCLab), we have to bite
+ // the bullet and do the standard parallel mark
+ _cm->markAndGrayObjectIfNecessary(obj);
+ }
+#if 1
+ if (_g1->isMarkedNext(old)) {
+ _cm->nextMarkBitMap()->parClear((HeapWord*)old);
+ }
+#endif
+ }
+
+ size_t* surv_young_words = _par_scan_state->surviving_young_words();
+ surv_young_words[young_index] += word_sz;
+
+ if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
+ arrayOop(old)->set_length(0);
+ _par_scan_state->push_on_queue((oop*) ((intptr_t)old | G1_PARTIAL_ARRAY_MASK));
+ } else {
+ _scanner->set_region(_g1->heap_region_containing(obj));
+ obj->oop_iterate_backwards(_scanner);
+ }
+ } else {
+ _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz);
+ obj = forward_ptr;
+ }
+ return obj;
+}
+
+template
+void G1ParCopyClosure::do_oop_work(oop* p) {
+ oop obj = *p;
+ assert(barrier != G1BarrierRS || obj != NULL,
+ "Precondition: G1BarrierRS implies obj is nonNull");
+
+ if (obj != NULL) {
+ if (_g1->obj_in_cs(obj)) {
+#if G1_REM_SET_LOGGING
+ gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" into CS.",
+ p, (void*) obj);
+#endif
+ if (obj->is_forwarded()) {
+ *p = obj->forwardee();
+ } else {
+ *p = copy_to_survivor_space(obj);
+ }
+ // When scanning the RS, we only care about objs in CS.
+ if (barrier == G1BarrierRS) {
+ _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+ }
+ }
+ // When scanning moved objs, must look at all oops.
+ if (barrier == G1BarrierEvac) {
+ _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+ }
+
+ if (do_gen_barrier) {
+ par_do_barrier(p);
+ }
+ }
+}
+
+template void G1ParCopyClosure::do_oop_work(oop* p);
+
+template void G1ParScanPartialArrayClosure::process_array_chunk(
+ oop obj, int start, int end) {
+ // process our set of indices (include header in first chunk)
+ assert(start < end, "invariant");
+ T* const base = (T*)objArrayOop(obj)->base();
+ T* const start_addr = base + start;
+ T* const end_addr = base + end;
+ MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
+ _scanner.set_region(_g1->heap_region_containing(obj));
+ obj->oop_iterate(&_scanner, mr);
+}
+
+void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
+ assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
+ oop old = oop((intptr_t)p & ~G1_PARTIAL_ARRAY_MASK);
+ assert(old->is_objArray(), "must be obj array");
+ assert(old->is_forwarded(), "must be forwarded");
+ assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
+
+ objArrayOop obj = objArrayOop(old->forwardee());
+ assert((void*)old != (void*)old->forwardee(), "self forwarding here?");
+ // Process ParGCArrayScanChunk elements now
+ // and push the remainder back onto queue
+ int start = arrayOop(old)->length();
+ int end = obj->length();
+ int remainder = end - start;
+ assert(start <= end, "just checking");
+ if (remainder > 2 * ParGCArrayScanChunk) {
+ // Test above combines last partial chunk with a full chunk
+ end = start + ParGCArrayScanChunk;
+ arrayOop(old)->set_length(end);
+ // Push remainder.
+ _par_scan_state->push_on_queue((oop*) ((intptr_t) old | G1_PARTIAL_ARRAY_MASK));
+ } else {
+ // Restore length so that the heap remains parsable in
+ // case of evacuation failure.
+ arrayOop(old)->set_length(end);
+ }
+
+ // process our set of indices (include header in first chunk)
+ process_array_chunk(obj, start, end);
+ oop* start_addr = start == 0 ? (oop*)obj : obj->obj_at_addr(start);
+ oop* end_addr = (oop*)(obj->base()) + end; // obj_at_addr(end) asserts end < length
+ MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
+ _scanner.set_region(_g1->heap_region_containing(obj));
+ obj->oop_iterate(&_scanner, mr);
+}
+
+int G1ScanAndBalanceClosure::_nq = 0;
+
+class G1ParEvacuateFollowersClosure : public VoidClosure {
+protected:
+ G1CollectedHeap* _g1h;
+ G1ParScanThreadState* _par_scan_state;
+ RefToScanQueueSet* _queues;
+ ParallelTaskTerminator* _terminator;
+
+ G1ParScanThreadState* par_scan_state() { return _par_scan_state; }
+ RefToScanQueueSet* queues() { return _queues; }
+ ParallelTaskTerminator* terminator() { return _terminator; }
+
+public:
+ G1ParEvacuateFollowersClosure(G1CollectedHeap* g1h,
+ G1ParScanThreadState* par_scan_state,
+ RefToScanQueueSet* queues,
+ ParallelTaskTerminator* terminator)
+ : _g1h(g1h), _par_scan_state(par_scan_state),
+ _queues(queues), _terminator(terminator) {}
+
+ void do_void() {
+ G1ParScanThreadState* pss = par_scan_state();
+ while (true) {
+ oop* ref_to_scan;
+ pss->trim_queue();
+ IF_G1_DETAILED_STATS(pss->note_steal_attempt());
+ if (queues()->steal(pss->queue_num(),
+ pss->hash_seed(),
+ ref_to_scan)) {
+ IF_G1_DETAILED_STATS(pss->note_steal());
+ pss->push_on_queue(ref_to_scan);
+ continue;
+ }
+ pss->start_term_time();
+ if (terminator()->offer_termination()) break;
+ pss->end_term_time();
+ }
+ pss->end_term_time();
+ pss->retire_alloc_buffers();
+ }
+};
+
+class G1ParTask : public AbstractGangTask {
+protected:
+ G1CollectedHeap* _g1h;
+ RefToScanQueueSet *_queues;
+ ParallelTaskTerminator _terminator;
+
+ Mutex _stats_lock;
+ Mutex* stats_lock() { return &_stats_lock; }
+
+ size_t getNCards() {
+ return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1)
+ / G1BlockOffsetSharedArray::N_bytes;
+ }
+
+public:
+ G1ParTask(G1CollectedHeap* g1h, int workers, RefToScanQueueSet *task_queues)
+ : AbstractGangTask("G1 collection"),
+ _g1h(g1h),
+ _queues(task_queues),
+ _terminator(workers, _queues),
+ _stats_lock(Mutex::leaf, "parallel G1 stats lock", true)
+ {}
+
+ RefToScanQueueSet* queues() { return _queues; }
+
+ RefToScanQueue *work_queue(int i) {
+ return queues()->queue(i);
+ }
+
+ void work(int i) {
+ ResourceMark rm;
+ HandleMark hm;
+
+ G1ParScanThreadState pss(_g1h, i);
+ G1ParScanHeapEvacClosure scan_evac_cl(_g1h, &pss);
+ G1ParScanHeapEvacClosure evac_failure_cl(_g1h, &pss);
+ G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss);
+
+ pss.set_evac_closure(&scan_evac_cl);
+ pss.set_evac_failure_closure(&evac_failure_cl);
+ pss.set_partial_scan_closure(&partial_scan_cl);
+
+ G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss);
+ G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss);
+ G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss);
+ G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss);
+ G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss);
+ G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss);
+
+ OopsInHeapRegionClosure *scan_root_cl;
+ OopsInHeapRegionClosure *scan_perm_cl;
+ OopsInHeapRegionClosure *scan_so_cl;
+
+ if (_g1h->g1_policy()->should_initiate_conc_mark()) {
+ scan_root_cl = &scan_mark_root_cl;
+ scan_perm_cl = &scan_mark_perm_cl;
+ scan_so_cl = &scan_mark_heap_rs_cl;
+ } else {
+ scan_root_cl = &only_scan_root_cl;
+ scan_perm_cl = &only_scan_perm_cl;
+ scan_so_cl = &only_scan_heap_rs_cl;
+ }
+
+ pss.start_strong_roots();
+ _g1h->g1_process_strong_roots(/* not collecting perm */ false,
+ SharedHeap::SO_AllClasses,
+ scan_root_cl,
+ &only_scan_heap_rs_cl,
+ scan_so_cl,
+ scan_perm_cl,
+ i);
+ pss.end_strong_roots();
+ {
+ double start = os::elapsedTime();
+ G1ParEvacuateFollowersClosure evac(_g1h, &pss, _queues, &_terminator);
+ evac.do_void();
+ double elapsed_ms = (os::elapsedTime()-start)*1000.0;
+ double term_ms = pss.term_time()*1000.0;
+ _g1h->g1_policy()->record_obj_copy_time(i, elapsed_ms-term_ms);
+ _g1h->g1_policy()->record_termination_time(i, term_ms);
+ }
+ _g1h->update_surviving_young_words(pss.surviving_young_words()+1);
+
+ // Clean up any par-expanded rem sets.
+ HeapRegionRemSet::par_cleanup();
+
+ MutexLocker x(stats_lock());
+ if (ParallelGCVerbose) {
+ gclog_or_tty->print("Thread %d complete:\n", i);
+#if G1_DETAILED_STATS
+ gclog_or_tty->print(" Pushes: %7d Pops: %7d Overflows: %7d Steals %7d (in %d attempts)\n",
+ pss.pushes(),
+ pss.pops(),
+ pss.overflow_pushes(),
+ pss.steals(),
+ pss.steal_attempts());
+#endif
+ double elapsed = pss.elapsed();
+ double strong_roots = pss.strong_roots_time();
+ double term = pss.term_time();
+ gclog_or_tty->print(" Elapsed: %7.2f ms.\n"
+ " Strong roots: %7.2f ms (%6.2f%%)\n"
+ " Termination: %7.2f ms (%6.2f%%) (in %d entries)\n",
+ elapsed * 1000.0,
+ strong_roots * 1000.0, (strong_roots*100.0/elapsed),
+ term * 1000.0, (term*100.0/elapsed),
+ pss.term_attempts());
+ size_t total_waste = pss.alloc_buffer_waste() + pss.undo_waste();
+ gclog_or_tty->print(" Waste: %8dK\n"
+ " Alloc Buffer: %8dK\n"
+ " Undo: %8dK\n",
+ (total_waste * HeapWordSize) / K,
+ (pss.alloc_buffer_waste() * HeapWordSize) / K,
+ (pss.undo_waste() * HeapWordSize) / K);
+ }
+
+ assert(pss.refs_to_scan() == 0, "Task queue should be empty");
+ assert(pss.overflowed_refs_to_scan() == 0, "Overflow queue should be empty");
+ }
+};
+
+// *** Common G1 Evacuation Stuff
+
+class G1CountClosure: public OopsInHeapRegionClosure {
+public:
+ int n;
+ G1CountClosure() : n(0) {}
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+ void do_oop(oop* p) {
+ oop obj = *p;
+ assert(obj != NULL && G1CollectedHeap::heap()->obj_in_cs(obj),
+ "Rem set closure called on non-rem-set pointer.");
+ n++;
+ }
+};
+
+static G1CountClosure count_closure;
+
+void
+G1CollectedHeap::
+g1_process_strong_roots(bool collecting_perm_gen,
+ SharedHeap::ScanningOption so,
+ OopClosure* scan_non_heap_roots,
+ OopsInHeapRegionClosure* scan_rs,
+ OopsInHeapRegionClosure* scan_so,
+ OopsInGenClosure* scan_perm,
+ int worker_i) {
+ // First scan the strong roots, including the perm gen.
+ double ext_roots_start = os::elapsedTime();
+ double closure_app_time_sec = 0.0;
+
+ BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
+ BufferingOopsInGenClosure buf_scan_perm(scan_perm);
+ buf_scan_perm.set_generation(perm_gen());
+
+ process_strong_roots(collecting_perm_gen, so,
+ &buf_scan_non_heap_roots,
+ &buf_scan_perm);
+ // Finish up any enqueued closure apps.
+ buf_scan_non_heap_roots.done();
+ buf_scan_perm.done();
+ double ext_roots_end = os::elapsedTime();
+ g1_policy()->reset_obj_copy_time(worker_i);
+ double obj_copy_time_sec =
+ buf_scan_non_heap_roots.closure_app_seconds() +
+ buf_scan_perm.closure_app_seconds();
+ g1_policy()->record_obj_copy_time(worker_i, obj_copy_time_sec * 1000.0);
+ double ext_root_time_ms =
+ ((ext_roots_end - ext_roots_start) - obj_copy_time_sec) * 1000.0;
+ g1_policy()->record_ext_root_scan_time(worker_i, ext_root_time_ms);
+
+ // Scan strong roots in mark stack.
+ if (!_process_strong_tasks->is_task_claimed(G1H_PS_mark_stack_oops_do)) {
+ concurrent_mark()->oops_do(scan_non_heap_roots);
+ }
+ double mark_stack_scan_ms = (os::elapsedTime() - ext_roots_end) * 1000.0;
+ g1_policy()->record_mark_stack_scan_time(worker_i, mark_stack_scan_ms);
+
+ // XXX What should this be doing in the parallel case?
+ g1_policy()->record_collection_pause_end_CH_strong_roots();
+ if (G1VerifyRemSet) {
+ // :::: FIXME ::::
+ // The stupid remembered set doesn't know how to filter out dead
+ // objects, which the smart one does, and so when it is created
+ // and then compared the number of entries in each differs and
+ // the verification code fails.
+ guarantee(false, "verification code is broken, see note");
+
+ // Let's make sure that the current rem set agrees with the stupidest
+ // one possible!
+ bool refs_enabled = ref_processor()->discovery_enabled();
+ if (refs_enabled) ref_processor()->disable_discovery();
+ StupidG1RemSet stupid(this);
+ count_closure.n = 0;
+ stupid.oops_into_collection_set_do(&count_closure, worker_i);
+ int stupid_n = count_closure.n;
+ count_closure.n = 0;
+ g1_rem_set()->oops_into_collection_set_do(&count_closure, worker_i);
+ guarantee(count_closure.n == stupid_n, "Old and new rem sets differ.");
+ gclog_or_tty->print_cr("\nFound %d pointers in heap RS.", count_closure.n);
+ if (refs_enabled) ref_processor()->enable_discovery();
+ }
+ if (scan_so != NULL) {
+ scan_scan_only_set(scan_so, worker_i);
+ }
+ // Now scan the complement of the collection set.
+ if (scan_rs != NULL) {
+ g1_rem_set()->oops_into_collection_set_do(scan_rs, worker_i);
+ }
+ // Finish with the ref_processor roots.
+ if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) {
+ ref_processor()->oops_do(scan_non_heap_roots);
+ }
+ g1_policy()->record_collection_pause_end_G1_strong_roots();
+ _process_strong_tasks->all_tasks_completed();
+}
+
+void
+G1CollectedHeap::scan_scan_only_region(HeapRegion* r,
+ OopsInHeapRegionClosure* oc,
+ int worker_i) {
+ HeapWord* startAddr = r->bottom();
+ HeapWord* endAddr = r->used_region().end();
+
+ oc->set_region(r);
+
+ HeapWord* p = r->bottom();
+ HeapWord* t = r->top();
+ guarantee( p == r->next_top_at_mark_start(), "invariant" );
+ while (p < t) {
+ oop obj = oop(p);
+ p += obj->oop_iterate(oc);
+ }
+}
+
+void
+G1CollectedHeap::scan_scan_only_set(OopsInHeapRegionClosure* oc,
+ int worker_i) {
+ double start = os::elapsedTime();
+
+ BufferingOopsInHeapRegionClosure boc(oc);
+
+ FilterInHeapRegionAndIntoCSClosure scan_only(this, &boc);
+ FilterAndMarkInHeapRegionAndIntoCSClosure scan_and_mark(this, &boc, concurrent_mark());
+
+ OopsInHeapRegionClosure *foc;
+ if (g1_policy()->should_initiate_conc_mark())
+ foc = &scan_and_mark;
+ else
+ foc = &scan_only;
+
+ HeapRegion* hr;
+ int n = 0;
+ while ((hr = _young_list->par_get_next_scan_only_region()) != NULL) {
+ scan_scan_only_region(hr, foc, worker_i);
+ ++n;
+ }
+ boc.done();
+
+ double closure_app_s = boc.closure_app_seconds();
+ g1_policy()->record_obj_copy_time(worker_i, closure_app_s * 1000.0);
+ double ms = (os::elapsedTime() - start - closure_app_s)*1000.0;
+ g1_policy()->record_scan_only_time(worker_i, ms, n);
+}
+
+void
+G1CollectedHeap::g1_process_weak_roots(OopClosure* root_closure,
+ OopClosure* non_root_closure) {
+ SharedHeap::process_weak_roots(root_closure, non_root_closure);
+}
+
+
+class SaveMarksClosure: public HeapRegionClosure {
+public:
+ bool doHeapRegion(HeapRegion* r) {
+ r->save_marks();
+ return false;
+ }
+};
+
+void G1CollectedHeap::save_marks() {
+ if (ParallelGCThreads == 0) {
+ SaveMarksClosure sm;
+ heap_region_iterate(&sm);
+ }
+ // We do this even in the parallel case
+ perm_gen()->save_marks();
+}
+
+void G1CollectedHeap::evacuate_collection_set() {
+ set_evacuation_failed(false);
+
+ g1_rem_set()->prepare_for_oops_into_collection_set_do();
+ concurrent_g1_refine()->set_use_cache(false);
+ int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
+
+ set_par_threads(n_workers);
+ G1ParTask g1_par_task(this, n_workers, _task_queues);
+
+ init_for_evac_failure(NULL);
+
+ change_strong_roots_parity(); // In preparation for parallel strong roots.
+ rem_set()->prepare_for_younger_refs_iterate(true);
+ double start_par = os::elapsedTime();
+
+ if (ParallelGCThreads > 0) {
+ // The individual threads will set their evac-failure closures.
+ workers()->run_task(&g1_par_task);
+ } else {
+ g1_par_task.work(0);
+ }
+
+ double par_time = (os::elapsedTime() - start_par) * 1000.0;
+ g1_policy()->record_par_time(par_time);
+ set_par_threads(0);
+ // Is this the right thing to do here? We don't save marks
+ // on individual heap regions when we allocate from
+ // them in parallel, so this seems like the correct place for this.
+ all_alloc_regions_note_end_of_copying();
+ {
+ G1IsAliveClosure is_alive(this);
+ G1KeepAliveClosure keep_alive(this);
+ JNIHandles::weak_oops_do(&is_alive, &keep_alive);
+ }
+
+ g1_rem_set()->cleanup_after_oops_into_collection_set_do();
+ concurrent_g1_refine()->set_use_cache(true);
+
+ finalize_for_evac_failure();
+
+ // Must do this before removing self-forwarding pointers, which clears
+ // the per-region evac-failure flags.
+ concurrent_mark()->complete_marking_in_collection_set();
+
+ if (evacuation_failed()) {
+ remove_self_forwarding_pointers();
+
+ if (PrintGCDetails) {
+ gclog_or_tty->print(" (evacuation failed)");
+ } else if (PrintGC) {
+ gclog_or_tty->print("--");
+ }
+ }
+
+ COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
+}
+
+void G1CollectedHeap::free_region(HeapRegion* hr) {
+ size_t pre_used = 0;
+ size_t cleared_h_regions = 0;
+ size_t freed_regions = 0;
+ UncleanRegionList local_list;
+
+ HeapWord* start = hr->bottom();
+ HeapWord* end = hr->prev_top_at_mark_start();
+ size_t used_bytes = hr->used();
+ size_t live_bytes = hr->max_live_bytes();
+ if (used_bytes > 0) {
+ guarantee( live_bytes <= used_bytes, "invariant" );
+ } else {
+ guarantee( live_bytes == 0, "invariant" );
+ }
+
+ size_t garbage_bytes = used_bytes - live_bytes;
+ if (garbage_bytes > 0)
+ g1_policy()->decrease_known_garbage_bytes(garbage_bytes);
+
+ free_region_work(hr, pre_used, cleared_h_regions, freed_regions,
+ &local_list);
+ finish_free_region_work(pre_used, cleared_h_regions, freed_regions,
+ &local_list);
+}
+
+void
+G1CollectedHeap::free_region_work(HeapRegion* hr,
+ size_t& pre_used,
+ size_t& cleared_h_regions,
+ size_t& freed_regions,
+ UncleanRegionList* list,
+ bool par) {
+ assert(!hr->popular(), "should not free popular regions");
+ pre_used += hr->used();
+ if (hr->isHumongous()) {
+ assert(hr->startsHumongous(),
+ "Only the start of a humongous region should be freed.");
+ int ind = _hrs->find(hr);
+ assert(ind != -1, "Should have an index.");
+ // Clear the start region.
+ hr->hr_clear(par, true /*clear_space*/);
+ list->insert_before_head(hr);
+ cleared_h_regions++;
+ freed_regions++;
+ // Clear any continued regions.
+ ind++;
+ while ((size_t)ind < n_regions()) {
+ HeapRegion* hrc = _hrs->at(ind);
+ if (!hrc->continuesHumongous()) break;
+ // Otherwise, does continue the H region.
+ assert(hrc->humongous_start_region() == hr, "Huh?");
+ hrc->hr_clear(par, true /*clear_space*/);
+ cleared_h_regions++;
+ freed_regions++;
+ list->insert_before_head(hrc);
+ ind++;
+ }
+ } else {
+ hr->hr_clear(par, true /*clear_space*/);
+ list->insert_before_head(hr);
+ freed_regions++;
+ // If we're using clear2, this should not be enabled.
+ // assert(!hr->in_cohort(), "Can't be both free and in a cohort.");
+ }
+}
+
+void G1CollectedHeap::finish_free_region_work(size_t pre_used,
+ size_t cleared_h_regions,
+ size_t freed_regions,
+ UncleanRegionList* list) {
+ if (list != NULL && list->sz() > 0) {
+ prepend_region_list_on_unclean_list(list);
+ }
+ // Acquire a lock, if we're parallel, to update possibly-shared
+ // variables.
+ Mutex* lock = (n_par_threads() > 0) ? ParGCRareEvent_lock : NULL;
+ {
+ MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag);
+ _summary_bytes_used -= pre_used;
+ _num_humongous_regions -= (int) cleared_h_regions;
+ _free_regions += freed_regions;
+ }
+}
+
+
+void G1CollectedHeap::dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRegion* list) {
+ while (list != NULL) {
+ guarantee( list->is_young(), "invariant" );
+
+ HeapWord* bottom = list->bottom();
+ HeapWord* end = list->end();
+ MemRegion mr(bottom, end);
+ ct_bs->dirty(mr);
+
+ list = list->get_next_young_region();
+ }
+}
+
+void G1CollectedHeap::cleanUpCardTable() {
+ CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set());
+ double start = os::elapsedTime();
+
+ ct_bs->clear(_g1_committed);
+
+ // now, redirty the cards of the scan-only and survivor regions
+ // (it seemed faster to do it this way, instead of iterating over
+ // all regions and then clearing / dirtying as approprite)
+ dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region());
+ dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region());
+
+ double elapsed = os::elapsedTime() - start;
+ g1_policy()->record_clear_ct_time( elapsed * 1000.0);
+}
+
+
+void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) {
+ // First do any popular regions.
+ HeapRegion* hr;
+ while ((hr = popular_region_to_evac()) != NULL) {
+ evac_popular_region(hr);
+ }
+ // Now do heuristic pauses.
+ if (g1_policy()->should_do_collection_pause(word_size)) {
+ do_collection_pause();
+ }
+}
+
+void G1CollectedHeap::free_collection_set(HeapRegion* cs_head) {
+ double young_time_ms = 0.0;
+ double non_young_time_ms = 0.0;
+
+ G1CollectorPolicy* policy = g1_policy();
+
+ double start_sec = os::elapsedTime();
+ bool non_young = true;
+
+ HeapRegion* cur = cs_head;
+ int age_bound = -1;
+ size_t rs_lengths = 0;
+
+ while (cur != NULL) {
+ if (non_young) {
+ if (cur->is_young()) {
+ double end_sec = os::elapsedTime();
+ double elapsed_ms = (end_sec - start_sec) * 1000.0;
+ non_young_time_ms += elapsed_ms;
+
+ start_sec = os::elapsedTime();
+ non_young = false;
+ }
+ } else {
+ if (!cur->is_on_free_list()) {
+ double end_sec = os::elapsedTime();
+ double elapsed_ms = (end_sec - start_sec) * 1000.0;
+ young_time_ms += elapsed_ms;
+
+ start_sec = os::elapsedTime();
+ non_young = true;
+ }
+ }
+
+ rs_lengths += cur->rem_set()->occupied();
+
+ HeapRegion* next = cur->next_in_collection_set();
+ assert(cur->in_collection_set(), "bad CS");
+ cur->set_next_in_collection_set(NULL);
+ cur->set_in_collection_set(false);
+
+ if (cur->is_young()) {
+ int index = cur->young_index_in_cset();
+ guarantee( index != -1, "invariant" );
+ guarantee( (size_t)index < policy->young_cset_length(), "invariant" );
+ size_t words_survived = _surviving_young_words[index];
+ cur->record_surv_words_in_group(words_survived);
+ } else {
+ int index = cur->young_index_in_cset();
+ guarantee( index == -1, "invariant" );
+ }
+
+ assert( (cur->is_young() && cur->young_index_in_cset() > -1) ||
+ (!cur->is_young() && cur->young_index_in_cset() == -1),
+ "invariant" );
+
+ if (!cur->evacuation_failed()) {
+ // And the region is empty.
+ assert(!cur->is_empty(),
+ "Should not have empty regions in a CS.");
+ free_region(cur);
+ } else {
+ guarantee( !cur->is_scan_only(), "should not be scan only" );
+ cur->uninstall_surv_rate_group();
+ if (cur->is_young())
+ cur->set_young_index_in_cset(-1);
+ cur->set_not_young();
+ cur->set_evacuation_failed(false);
+ }
+ cur = next;
+ }
+
+ policy->record_max_rs_lengths(rs_lengths);
+ policy->cset_regions_freed();
+
+ double end_sec = os::elapsedTime();
+ double elapsed_ms = (end_sec - start_sec) * 1000.0;
+ if (non_young)
+ non_young_time_ms += elapsed_ms;
+ else
+ young_time_ms += elapsed_ms;
+
+ policy->record_young_free_cset_time_ms(young_time_ms);
+ policy->record_non_young_free_cset_time_ms(non_young_time_ms);
+}
+
+HeapRegion*
+G1CollectedHeap::alloc_region_from_unclean_list_locked(bool zero_filled) {
+ assert(ZF_mon->owned_by_self(), "Precondition");
+ HeapRegion* res = pop_unclean_region_list_locked();
+ if (res != NULL) {
+ assert(!res->continuesHumongous() &&
+ res->zero_fill_state() != HeapRegion::Allocated,
+ "Only free regions on unclean list.");
+ if (zero_filled) {
+ res->ensure_zero_filled_locked();
+ res->set_zero_fill_allocated();
+ }
+ }
+ return res;
+}
+
+HeapRegion* G1CollectedHeap::alloc_region_from_unclean_list(bool zero_filled) {
+ MutexLockerEx zx(ZF_mon, Mutex::_no_safepoint_check_flag);
+ return alloc_region_from_unclean_list_locked(zero_filled);
+}
+
+void G1CollectedHeap::put_region_on_unclean_list(HeapRegion* r) {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ put_region_on_unclean_list_locked(r);
+ if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread.
+}
+
+void G1CollectedHeap::set_unclean_regions_coming(bool b) {
+ MutexLockerEx x(Cleanup_mon);
+ set_unclean_regions_coming_locked(b);
+}
+
+void G1CollectedHeap::set_unclean_regions_coming_locked(bool b) {
+ assert(Cleanup_mon->owned_by_self(), "Precondition");
+ _unclean_regions_coming = b;
+ // Wake up mutator threads that might be waiting for completeCleanup to
+ // finish.
+ if (!b) Cleanup_mon->notify_all();
+}
+
+void G1CollectedHeap::wait_for_cleanup_complete() {
+ MutexLockerEx x(Cleanup_mon);
+ wait_for_cleanup_complete_locked();
+}
+
+void G1CollectedHeap::wait_for_cleanup_complete_locked() {
+ assert(Cleanup_mon->owned_by_self(), "precondition");
+ while (_unclean_regions_coming) {
+ Cleanup_mon->wait();
+ }
+}
+
+void
+G1CollectedHeap::put_region_on_unclean_list_locked(HeapRegion* r) {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ _unclean_region_list.insert_before_head(r);
+}
+
+void
+G1CollectedHeap::prepend_region_list_on_unclean_list(UncleanRegionList* list) {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ prepend_region_list_on_unclean_list_locked(list);
+ if (should_zf()) ZF_mon->notify_all(); // Wake up ZF thread.
+}
+
+void
+G1CollectedHeap::
+prepend_region_list_on_unclean_list_locked(UncleanRegionList* list) {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ _unclean_region_list.prepend_list(list);
+}
+
+HeapRegion* G1CollectedHeap::pop_unclean_region_list_locked() {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ HeapRegion* res = _unclean_region_list.pop();
+ if (res != NULL) {
+ // Inform ZF thread that there's a new unclean head.
+ if (_unclean_region_list.hd() != NULL && should_zf())
+ ZF_mon->notify_all();
+ }
+ return res;
+}
+
+HeapRegion* G1CollectedHeap::peek_unclean_region_list_locked() {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ return _unclean_region_list.hd();
+}
+
+
+bool G1CollectedHeap::move_cleaned_region_to_free_list_locked() {
+ assert(ZF_mon->owned_by_self(), "Precondition");
+ HeapRegion* r = peek_unclean_region_list_locked();
+ if (r != NULL && r->zero_fill_state() == HeapRegion::ZeroFilled) {
+ // Result of below must be equal to "r", since we hold the lock.
+ (void)pop_unclean_region_list_locked();
+ put_free_region_on_list_locked(r);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+bool G1CollectedHeap::move_cleaned_region_to_free_list() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ return move_cleaned_region_to_free_list_locked();
+}
+
+
+void G1CollectedHeap::put_free_region_on_list_locked(HeapRegion* r) {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ assert(_free_region_list_size == free_region_list_length(), "Inv");
+ assert(r->zero_fill_state() == HeapRegion::ZeroFilled,
+ "Regions on free list must be zero filled");
+ assert(!r->isHumongous(), "Must not be humongous.");
+ assert(r->is_empty(), "Better be empty");
+ assert(!r->is_on_free_list(),
+ "Better not already be on free list");
+ assert(!r->is_on_unclean_list(),
+ "Better not already be on unclean list");
+ r->set_on_free_list(true);
+ r->set_next_on_free_list(_free_region_list);
+ _free_region_list = r;
+ _free_region_list_size++;
+ assert(_free_region_list_size == free_region_list_length(), "Inv");
+}
+
+void G1CollectedHeap::put_free_region_on_list(HeapRegion* r) {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ put_free_region_on_list_locked(r);
+}
+
+HeapRegion* G1CollectedHeap::pop_free_region_list_locked() {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ assert(_free_region_list_size == free_region_list_length(), "Inv");
+ HeapRegion* res = _free_region_list;
+ if (res != NULL) {
+ _free_region_list = res->next_from_free_list();
+ _free_region_list_size--;
+ res->set_on_free_list(false);
+ res->set_next_on_free_list(NULL);
+ assert(_free_region_list_size == free_region_list_length(), "Inv");
+ }
+ return res;
+}
+
+
+HeapRegion* G1CollectedHeap::alloc_free_region_from_lists(bool zero_filled) {
+ // By self, or on behalf of self.
+ assert(Heap_lock->is_locked(), "Precondition");
+ HeapRegion* res = NULL;
+ bool first = true;
+ while (res == NULL) {
+ if (zero_filled || !first) {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ res = pop_free_region_list_locked();
+ if (res != NULL) {
+ assert(!res->zero_fill_is_allocated(),
+ "No allocated regions on free list.");
+ res->set_zero_fill_allocated();
+ } else if (!first) {
+ break; // We tried both, time to return NULL.
+ }
+ }
+
+ if (res == NULL) {
+ res = alloc_region_from_unclean_list(zero_filled);
+ }
+ assert(res == NULL ||
+ !zero_filled ||
+ res->zero_fill_is_allocated(),
+ "We must have allocated the region we're returning");
+ first = false;
+ }
+ return res;
+}
+
+void G1CollectedHeap::remove_allocated_regions_from_lists() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ {
+ HeapRegion* prev = NULL;
+ HeapRegion* cur = _unclean_region_list.hd();
+ while (cur != NULL) {
+ HeapRegion* next = cur->next_from_unclean_list();
+ if (cur->zero_fill_is_allocated()) {
+ // Remove from the list.
+ if (prev == NULL) {
+ (void)_unclean_region_list.pop();
+ } else {
+ _unclean_region_list.delete_after(prev);
+ }
+ cur->set_on_unclean_list(false);
+ cur->set_next_on_unclean_list(NULL);
+ } else {
+ prev = cur;
+ }
+ cur = next;
+ }
+ assert(_unclean_region_list.sz() == unclean_region_list_length(),
+ "Inv");
+ }
+
+ {
+ HeapRegion* prev = NULL;
+ HeapRegion* cur = _free_region_list;
+ while (cur != NULL) {
+ HeapRegion* next = cur->next_from_free_list();
+ if (cur->zero_fill_is_allocated()) {
+ // Remove from the list.
+ if (prev == NULL) {
+ _free_region_list = cur->next_from_free_list();
+ } else {
+ prev->set_next_on_free_list(cur->next_from_free_list());
+ }
+ cur->set_on_free_list(false);
+ cur->set_next_on_free_list(NULL);
+ _free_region_list_size--;
+ } else {
+ prev = cur;
+ }
+ cur = next;
+ }
+ assert(_free_region_list_size == free_region_list_length(), "Inv");
+ }
+}
+
+bool G1CollectedHeap::verify_region_lists() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ return verify_region_lists_locked();
+}
+
+bool G1CollectedHeap::verify_region_lists_locked() {
+ HeapRegion* unclean = _unclean_region_list.hd();
+ while (unclean != NULL) {
+ guarantee(unclean->is_on_unclean_list(), "Well, it is!");
+ guarantee(!unclean->is_on_free_list(), "Well, it shouldn't be!");
+ guarantee(unclean->zero_fill_state() != HeapRegion::Allocated,
+ "Everything else is possible.");
+ unclean = unclean->next_from_unclean_list();
+ }
+ guarantee(_unclean_region_list.sz() == unclean_region_list_length(), "Inv");
+
+ HeapRegion* free_r = _free_region_list;
+ while (free_r != NULL) {
+ assert(free_r->is_on_free_list(), "Well, it is!");
+ assert(!free_r->is_on_unclean_list(), "Well, it shouldn't be!");
+ switch (free_r->zero_fill_state()) {
+ case HeapRegion::NotZeroFilled:
+ case HeapRegion::ZeroFilling:
+ guarantee(false, "Should not be on free list.");
+ break;
+ default:
+ // Everything else is possible.
+ break;
+ }
+ free_r = free_r->next_from_free_list();
+ }
+ guarantee(_free_region_list_size == free_region_list_length(), "Inv");
+ // If we didn't do an assertion...
+ return true;
+}
+
+size_t G1CollectedHeap::free_region_list_length() {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ size_t len = 0;
+ HeapRegion* cur = _free_region_list;
+ while (cur != NULL) {
+ len++;
+ cur = cur->next_from_free_list();
+ }
+ return len;
+}
+
+size_t G1CollectedHeap::unclean_region_list_length() {
+ assert(ZF_mon->owned_by_self(), "precondition.");
+ return _unclean_region_list.length();
+}
+
+size_t G1CollectedHeap::n_regions() {
+ return _hrs->length();
+}
+
+size_t G1CollectedHeap::max_regions() {
+ return
+ (size_t)align_size_up(g1_reserved_obj_bytes(), HeapRegion::GrainBytes) /
+ HeapRegion::GrainBytes;
+}
+
+size_t G1CollectedHeap::free_regions() {
+ /* Possibly-expensive assert.
+ assert(_free_regions == count_free_regions(),
+ "_free_regions is off.");
+ */
+ return _free_regions;
+}
+
+bool G1CollectedHeap::should_zf() {
+ return _free_region_list_size < (size_t) G1ConcZFMaxRegions;
+}
+
+class RegionCounter: public HeapRegionClosure {
+ size_t _n;
+public:
+ RegionCounter() : _n(0) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->is_empty() && !r->popular()) {
+ assert(!r->isHumongous(), "H regions should not be empty.");
+ _n++;
+ }
+ return false;
+ }
+ int res() { return (int) _n; }
+};
+
+size_t G1CollectedHeap::count_free_regions() {
+ RegionCounter rc;
+ heap_region_iterate(&rc);
+ size_t n = rc.res();
+ if (_cur_alloc_region != NULL && _cur_alloc_region->is_empty())
+ n--;
+ return n;
+}
+
+size_t G1CollectedHeap::count_free_regions_list() {
+ size_t n = 0;
+ size_t o = 0;
+ ZF_mon->lock_without_safepoint_check();
+ HeapRegion* cur = _free_region_list;
+ while (cur != NULL) {
+ cur = cur->next_from_free_list();
+ n++;
+ }
+ size_t m = unclean_region_list_length();
+ ZF_mon->unlock();
+ return n + m;
+}
+
+bool G1CollectedHeap::should_set_young_locked() {
+ assert(heap_lock_held_for_gc(),
+ "the heap lock should already be held by or for this thread");
+ return (g1_policy()->in_young_gc_mode() &&
+ g1_policy()->should_add_next_region_to_young_list());
+}
+
+void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) {
+ assert(heap_lock_held_for_gc(),
+ "the heap lock should already be held by or for this thread");
+ _young_list->push_region(hr);
+ g1_policy()->set_region_short_lived(hr);
+}
+
+class NoYoungRegionsClosure: public HeapRegionClosure {
+private:
+ bool _success;
+public:
+ NoYoungRegionsClosure() : _success(true) { }
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->is_young()) {
+ gclog_or_tty->print_cr("Region ["PTR_FORMAT", "PTR_FORMAT") tagged as young",
+ r->bottom(), r->end());
+ _success = false;
+ }
+ return false;
+ }
+ bool success() { return _success; }
+};
+
+bool G1CollectedHeap::check_young_list_empty(bool ignore_scan_only_list,
+ bool check_sample) {
+ bool ret = true;
+
+ ret = _young_list->check_list_empty(ignore_scan_only_list, check_sample);
+ if (!ignore_scan_only_list) {
+ NoYoungRegionsClosure closure;
+ heap_region_iterate(&closure);
+ ret = ret && closure.success();
+ }
+
+ return ret;
+}
+
+void G1CollectedHeap::empty_young_list() {
+ assert(heap_lock_held_for_gc(),
+ "the heap lock should already be held by or for this thread");
+ assert(g1_policy()->in_young_gc_mode(), "should be in young GC mode");
+
+ _young_list->empty_list();
+}
+
+bool G1CollectedHeap::all_alloc_regions_no_allocs_since_save_marks() {
+ bool no_allocs = true;
+ for (int ap = 0; ap < GCAllocPurposeCount && no_allocs; ++ap) {
+ HeapRegion* r = _gc_alloc_regions[ap];
+ no_allocs = r == NULL || r->saved_mark_at_top();
+ }
+ return no_allocs;
+}
+
+void G1CollectedHeap::all_alloc_regions_note_end_of_copying() {
+ for (int ap = 0; ap < GCAllocPurposeCount; ++ap) {
+ HeapRegion* r = _gc_alloc_regions[ap];
+ if (r != NULL) {
+ // Check for aliases.
+ bool has_processed_alias = false;
+ for (int i = 0; i < ap; ++i) {
+ if (_gc_alloc_regions[i] == r) {
+ has_processed_alias = true;
+ break;
+ }
+ }
+ if (!has_processed_alias) {
+ r->note_end_of_copying();
+ g1_policy()->record_after_bytes(r->used());
+ }
+ }
+ }
+}
+
+
+// Done at the start of full GC.
+void G1CollectedHeap::tear_down_region_lists() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ while (pop_unclean_region_list_locked() != NULL) ;
+ assert(_unclean_region_list.hd() == NULL && _unclean_region_list.sz() == 0,
+ "Postconditions of loop.")
+ while (pop_free_region_list_locked() != NULL) ;
+ assert(_free_region_list == NULL, "Postcondition of loop.");
+ if (_free_region_list_size != 0) {
+ gclog_or_tty->print_cr("Size is %d.", _free_region_list_size);
+ print();
+ }
+ assert(_free_region_list_size == 0, "Postconditions of loop.");
+}
+
+
+class RegionResetter: public HeapRegionClosure {
+ G1CollectedHeap* _g1;
+ int _n;
+public:
+ RegionResetter() : _g1(G1CollectedHeap::heap()), _n(0) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->continuesHumongous()) return false;
+ if (r->top() > r->bottom()) {
+ if (r->top() < r->end()) {
+ Copy::fill_to_words(r->top(),
+ pointer_delta(r->end(), r->top()));
+ }
+ r->set_zero_fill_allocated();
+ } else {
+ assert(r->is_empty(), "tautology");
+ if (r->popular()) {
+ if (r->zero_fill_state() != HeapRegion::Allocated) {
+ r->ensure_zero_filled_locked();
+ r->set_zero_fill_allocated();
+ }
+ } else {
+ _n++;
+ switch (r->zero_fill_state()) {
+ case HeapRegion::NotZeroFilled:
+ case HeapRegion::ZeroFilling:
+ _g1->put_region_on_unclean_list_locked(r);
+ break;
+ case HeapRegion::Allocated:
+ r->set_zero_fill_complete();
+ // no break; go on to put on free list.
+ case HeapRegion::ZeroFilled:
+ _g1->put_free_region_on_list_locked(r);
+ break;
+ }
+ }
+ }
+ return false;
+ }
+
+ int getFreeRegionCount() {return _n;}
+};
+
+// Done at the end of full GC.
+void G1CollectedHeap::rebuild_region_lists() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ // This needs to go at the end of the full GC.
+ RegionResetter rs;
+ heap_region_iterate(&rs);
+ _free_regions = rs.getFreeRegionCount();
+ // Tell the ZF thread it may have work to do.
+ if (should_zf()) ZF_mon->notify_all();
+}
+
+class UsedRegionsNeedZeroFillSetter: public HeapRegionClosure {
+ G1CollectedHeap* _g1;
+ int _n;
+public:
+ UsedRegionsNeedZeroFillSetter() : _g1(G1CollectedHeap::heap()), _n(0) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (r->continuesHumongous()) return false;
+ if (r->top() > r->bottom()) {
+ // There are assertions in "set_zero_fill_needed()" below that
+ // require top() == bottom(), so this is technically illegal.
+ // We'll skirt the law here, by making that true temporarily.
+ DEBUG_ONLY(HeapWord* save_top = r->top();
+ r->set_top(r->bottom()));
+ r->set_zero_fill_needed();
+ DEBUG_ONLY(r->set_top(save_top));
+ }
+ return false;
+ }
+};
+
+// Done at the start of full GC.
+void G1CollectedHeap::set_used_regions_to_need_zero_fill() {
+ MutexLockerEx x(ZF_mon, Mutex::_no_safepoint_check_flag);
+ // This needs to go at the end of the full GC.
+ UsedRegionsNeedZeroFillSetter rs;
+ heap_region_iterate(&rs);
+}
+
+class CountObjClosure: public ObjectClosure {
+ size_t _n;
+public:
+ CountObjClosure() : _n(0) {}
+ void do_object(oop obj) { _n++; }
+ size_t n() { return _n; }
+};
+
+size_t G1CollectedHeap::pop_object_used_objs() {
+ size_t sum_objs = 0;
+ for (int i = 0; i < G1NumPopularRegions; i++) {
+ CountObjClosure cl;
+ _hrs->at(i)->object_iterate(&cl);
+ sum_objs += cl.n();
+ }
+ return sum_objs;
+}
+
+size_t G1CollectedHeap::pop_object_used_bytes() {
+ size_t sum_bytes = 0;
+ for (int i = 0; i < G1NumPopularRegions; i++) {
+ sum_bytes += _hrs->at(i)->used();
+ }
+ return sum_bytes;
+}
+
+
+static int nq = 0;
+
+HeapWord* G1CollectedHeap::allocate_popular_object(size_t word_size) {
+ while (_cur_pop_hr_index < G1NumPopularRegions) {
+ HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index);
+ HeapWord* res = cur_pop_region->allocate(word_size);
+ if (res != NULL) {
+ // We account for popular objs directly in the used summary:
+ _summary_bytes_used += (word_size * HeapWordSize);
+ return res;
+ }
+ // Otherwise, try the next region (first making sure that we remember
+ // the last "top" value as the "next_top_at_mark_start", so that
+ // objects made popular during markings aren't automatically considered
+ // live).
+ cur_pop_region->note_end_of_copying();
+ // Otherwise, try the next region.
+ _cur_pop_hr_index++;
+ }
+ // XXX: For now !!!
+ vm_exit_out_of_memory(word_size,
+ "Not enough pop obj space (To Be Fixed)");
+ return NULL;
+}
+
+class HeapRegionList: public CHeapObj {
+ public:
+ HeapRegion* hr;
+ HeapRegionList* next;
+};
+
+void G1CollectedHeap::schedule_popular_region_evac(HeapRegion* r) {
+ // This might happen during parallel GC, so protect by this lock.
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ // We don't schedule regions whose evacuations are already pending, or
+ // are already being evacuated.
+ if (!r->popular_pending() && !r->in_collection_set()) {
+ r->set_popular_pending(true);
+ if (G1TracePopularity) {
+ gclog_or_tty->print_cr("Scheduling region "PTR_FORMAT" "
+ "["PTR_FORMAT", "PTR_FORMAT") for pop-object evacuation.",
+ r, r->bottom(), r->end());
+ }
+ HeapRegionList* hrl = new HeapRegionList;
+ hrl->hr = r;
+ hrl->next = _popular_regions_to_be_evacuated;
+ _popular_regions_to_be_evacuated = hrl;
+ }
+}
+
+HeapRegion* G1CollectedHeap::popular_region_to_evac() {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ HeapRegion* res = NULL;
+ while (_popular_regions_to_be_evacuated != NULL && res == NULL) {
+ HeapRegionList* hrl = _popular_regions_to_be_evacuated;
+ _popular_regions_to_be_evacuated = hrl->next;
+ res = hrl->hr;
+ // The G1RSPopLimit may have increased, so recheck here...
+ if (res->rem_set()->occupied() < (size_t) G1RSPopLimit) {
+ // Hah: don't need to schedule.
+ if (G1TracePopularity) {
+ gclog_or_tty->print_cr("Unscheduling region "PTR_FORMAT" "
+ "["PTR_FORMAT", "PTR_FORMAT") "
+ "for pop-object evacuation (size %d < limit %d)",
+ res, res->bottom(), res->end(),
+ res->rem_set()->occupied(), G1RSPopLimit);
+ }
+ res->set_popular_pending(false);
+ res = NULL;
+ }
+ // We do not reset res->popular() here; if we did so, it would allow
+ // the region to be "rescheduled" for popularity evacuation. Instead,
+ // this is done in the collection pause, with the world stopped.
+ // So the invariant is that the regions in the list have the popularity
+ // boolean set, but having the boolean set does not imply membership
+ // on the list (though there can at most one such pop-pending region
+ // not on the list at any time).
+ delete hrl;
+ }
+ return res;
+}
+
+void G1CollectedHeap::evac_popular_region(HeapRegion* hr) {
+ while (true) {
+ // Don't want to do a GC pause while cleanup is being completed!
+ wait_for_cleanup_complete();
+
+ // Read the GC count while holding the Heap_lock
+ int gc_count_before = SharedHeap::heap()->total_collections();
+ g1_policy()->record_stop_world_start();
+
+ {
+ MutexUnlocker mu(Heap_lock); // give up heap lock, execute gets it back
+ VM_G1PopRegionCollectionPause op(gc_count_before, hr);
+ VMThread::execute(&op);
+
+ // If the prolog succeeded, we didn't do a GC for this.
+ if (op.prologue_succeeded()) break;
+ }
+ // Otherwise we didn't. We should recheck the size, though, since
+ // the limit may have increased...
+ if (hr->rem_set()->occupied() < (size_t) G1RSPopLimit) {
+ hr->set_popular_pending(false);
+ break;
+ }
+ }
+}
+
+void G1CollectedHeap::atomic_inc_obj_rc(oop obj) {
+ Atomic::inc(obj_rc_addr(obj));
+}
+
+class CountRCClosure: public OopsInHeapRegionClosure {
+ G1CollectedHeap* _g1h;
+ bool _parallel;
+public:
+ CountRCClosure(G1CollectedHeap* g1h) :
+ _g1h(g1h), _parallel(ParallelGCThreads > 0)
+ {}
+ void do_oop(narrowOop* p) {
+ guarantee(false, "NYI");
+ }
+ void do_oop(oop* p) {
+ oop obj = *p;
+ assert(obj != NULL, "Precondition.");
+ if (_parallel) {
+ // We go sticky at the limit to avoid excess contention.
+ // If we want to track the actual RC's further, we'll need to keep a
+ // per-thread hash table or something for the popular objects.
+ if (_g1h->obj_rc(obj) < G1ObjPopLimit) {
+ _g1h->atomic_inc_obj_rc(obj);
+ }
+ } else {
+ _g1h->inc_obj_rc(obj);
+ }
+ }
+};
+
+class EvacPopObjClosure: public ObjectClosure {
+ G1CollectedHeap* _g1h;
+ size_t _pop_objs;
+ size_t _max_rc;
+public:
+ EvacPopObjClosure(G1CollectedHeap* g1h) :
+ _g1h(g1h), _pop_objs(0), _max_rc(0) {}
+
+ void do_object(oop obj) {
+ size_t rc = _g1h->obj_rc(obj);
+ _max_rc = MAX2(rc, _max_rc);
+ if (rc >= (size_t) G1ObjPopLimit) {
+ _g1h->_pop_obj_rc_at_copy.add((double)rc);
+ size_t word_sz = obj->size();
+ HeapWord* new_pop_loc = _g1h->allocate_popular_object(word_sz);
+ oop new_pop_obj = (oop)new_pop_loc;
+ Copy::aligned_disjoint_words((HeapWord*)obj, new_pop_loc, word_sz);
+ obj->forward_to(new_pop_obj);
+ G1ScanAndBalanceClosure scan_and_balance(_g1h);
+ new_pop_obj->oop_iterate_backwards(&scan_and_balance);
+ // preserve "next" mark bit if marking is in progress.
+ if (_g1h->mark_in_progress() && !_g1h->is_obj_ill(obj)) {
+ _g1h->concurrent_mark()->markAndGrayObjectIfNecessary(new_pop_obj);
+ }
+
+ if (G1TracePopularity) {
+ gclog_or_tty->print_cr("Found obj " PTR_FORMAT " of word size " SIZE_FORMAT
+ " pop (%d), move to " PTR_FORMAT,
+ (void*) obj, word_sz,
+ _g1h->obj_rc(obj), (void*) new_pop_obj);
+ }
+ _pop_objs++;
+ }
+ }
+ size_t pop_objs() { return _pop_objs; }
+ size_t max_rc() { return _max_rc; }
+};
+
+class G1ParCountRCTask : public AbstractGangTask {
+ G1CollectedHeap* _g1h;
+ BitMap _bm;
+
+ size_t getNCards() {
+ return (_g1h->capacity() + G1BlockOffsetSharedArray::N_bytes - 1)
+ / G1BlockOffsetSharedArray::N_bytes;
+ }
+ CountRCClosure _count_rc_closure;
+public:
+ G1ParCountRCTask(G1CollectedHeap* g1h) :
+ AbstractGangTask("G1 Par RC Count task"),
+ _g1h(g1h), _bm(getNCards()), _count_rc_closure(g1h)
+ {}
+
+ void work(int i) {
+ ResourceMark rm;
+ HandleMark hm;
+ _g1h->g1_rem_set()->oops_into_collection_set_do(&_count_rc_closure, i);
+ }
+};
+
+void G1CollectedHeap::popularity_pause_preamble(HeapRegion* popular_region) {
+ // We're evacuating a single region (for popularity).
+ if (G1TracePopularity) {
+ gclog_or_tty->print_cr("Doing pop region pause for ["PTR_FORMAT", "PTR_FORMAT")",
+ popular_region->bottom(), popular_region->end());
+ }
+ g1_policy()->set_single_region_collection_set(popular_region);
+ size_t max_rc;
+ if (!compute_reference_counts_and_evac_popular(popular_region,
+ &max_rc)) {
+ // We didn't evacuate any popular objects.
+ // We increase the RS popularity limit, to prevent this from
+ // happening in the future.
+ if (G1RSPopLimit < (1 << 30)) {
+ G1RSPopLimit *= 2;
+ }
+ // For now, interesting enough for a message:
+#if 1
+ gclog_or_tty->print_cr("In pop region pause for ["PTR_FORMAT", "PTR_FORMAT"), "
+ "failed to find a pop object (max = %d).",
+ popular_region->bottom(), popular_region->end(),
+ max_rc);
+ gclog_or_tty->print_cr("Increased G1RSPopLimit to %d.", G1RSPopLimit);
+#endif // 0
+ // Also, we reset the collection set to NULL, to make the rest of
+ // the collection do nothing.
+ assert(popular_region->next_in_collection_set() == NULL,
+ "should be single-region.");
+ popular_region->set_in_collection_set(false);
+ popular_region->set_popular_pending(false);
+ g1_policy()->clear_collection_set();
+ }
+}
+
+bool G1CollectedHeap::
+compute_reference_counts_and_evac_popular(HeapRegion* popular_region,
+ size_t* max_rc) {
+ HeapWord* rc_region_bot;
+ HeapWord* rc_region_end;
+
+ // Set up the reference count region.
+ HeapRegion* rc_region = newAllocRegion(HeapRegion::GrainWords);
+ if (rc_region != NULL) {
+ rc_region_bot = rc_region->bottom();
+ rc_region_end = rc_region->end();
+ } else {
+ rc_region_bot = NEW_C_HEAP_ARRAY(HeapWord, HeapRegion::GrainWords);
+ if (rc_region_bot == NULL) {
+ vm_exit_out_of_memory(HeapRegion::GrainWords,
+ "No space for RC region.");
+ }
+ rc_region_end = rc_region_bot + HeapRegion::GrainWords;
+ }
+
+ if (G1TracePopularity)
+ gclog_or_tty->print_cr("RC region is ["PTR_FORMAT", "PTR_FORMAT")",
+ rc_region_bot, rc_region_end);
+ if (rc_region_bot > popular_region->bottom()) {
+ _rc_region_above = true;
+ _rc_region_diff =
+ pointer_delta(rc_region_bot, popular_region->bottom(), 1);
+ } else {
+ assert(rc_region_bot < popular_region->bottom(), "Can't be equal.");
+ _rc_region_above = false;
+ _rc_region_diff =
+ pointer_delta(popular_region->bottom(), rc_region_bot, 1);
+ }
+ g1_policy()->record_pop_compute_rc_start();
+ // Count external references.
+ g1_rem_set()->prepare_for_oops_into_collection_set_do();
+ if (ParallelGCThreads > 0) {
+
+ set_par_threads(workers()->total_workers());
+ G1ParCountRCTask par_count_rc_task(this);
+ workers()->run_task(&par_count_rc_task);
+ set_par_threads(0);
+
+ } else {
+ CountRCClosure count_rc_closure(this);
+ g1_rem_set()->oops_into_collection_set_do(&count_rc_closure, 0);
+ }
+ g1_rem_set()->cleanup_after_oops_into_collection_set_do();
+ g1_policy()->record_pop_compute_rc_end();
+
+ // Now evacuate popular objects.
+ g1_policy()->record_pop_evac_start();
+ EvacPopObjClosure evac_pop_obj_cl(this);
+ popular_region->object_iterate(&evac_pop_obj_cl);
+ *max_rc = evac_pop_obj_cl.max_rc();
+
+ // Make sure the last "top" value of the current popular region is copied
+ // as the "next_top_at_mark_start", so that objects made popular during
+ // markings aren't automatically considered live.
+ HeapRegion* cur_pop_region = _hrs->at(_cur_pop_hr_index);
+ cur_pop_region->note_end_of_copying();
+
+ if (rc_region != NULL) {
+ free_region(rc_region);
+ } else {
+ FREE_C_HEAP_ARRAY(HeapWord, rc_region_bot);
+ }
+ g1_policy()->record_pop_evac_end();
+
+ return evac_pop_obj_cl.pop_objs() > 0;
+}
+
+class CountPopObjInfoClosure: public HeapRegionClosure {
+ size_t _objs;
+ size_t _bytes;
+
+ class CountObjClosure: public ObjectClosure {
+ int _n;
+ public:
+ CountObjClosure() : _n(0) {}
+ void do_object(oop obj) { _n++; }
+ size_t n() { return _n; }
+ };
+
+public:
+ CountPopObjInfoClosure() : _objs(0), _bytes(0) {}
+ bool doHeapRegion(HeapRegion* r) {
+ _bytes += r->used();
+ CountObjClosure blk;
+ r->object_iterate(&blk);
+ _objs += blk.n();
+ return false;
+ }
+ size_t objs() { return _objs; }
+ size_t bytes() { return _bytes; }
+};
+
+
+void G1CollectedHeap::print_popularity_summary_info() const {
+ CountPopObjInfoClosure blk;
+ for (int i = 0; i <= _cur_pop_hr_index; i++) {
+ blk.doHeapRegion(_hrs->at(i));
+ }
+ gclog_or_tty->print_cr("\nPopular objects: %d objs, %d bytes.",
+ blk.objs(), blk.bytes());
+ gclog_or_tty->print_cr(" RC at copy = [avg = %5.2f, max = %5.2f, sd = %5.2f].",
+ _pop_obj_rc_at_copy.avg(),
+ _pop_obj_rc_at_copy.maximum(),
+ _pop_obj_rc_at_copy.sd());
+}
+
+void G1CollectedHeap::set_refine_cte_cl_concurrency(bool concurrent) {
+ _refine_cte_cl->set_concurrent(concurrent);
+}
+
+#ifndef PRODUCT
+
+class PrintHeapRegionClosure: public HeapRegionClosure {
+public:
+ bool doHeapRegion(HeapRegion *r) {
+ gclog_or_tty->print("Region: "PTR_FORMAT":", r);
+ if (r != NULL) {
+ if (r->is_on_free_list())
+ gclog_or_tty->print("Free ");
+ if (r->is_young())
+ gclog_or_tty->print("Young ");
+ if (r->isHumongous())
+ gclog_or_tty->print("Is Humongous ");
+ r->print();
+ }
+ return false;
+ }
+};
+
+class SortHeapRegionClosure : public HeapRegionClosure {
+ size_t young_regions,free_regions, unclean_regions;
+ size_t hum_regions, count;
+ size_t unaccounted, cur_unclean, cur_alloc;
+ size_t total_free;
+ HeapRegion* cur;
+public:
+ SortHeapRegionClosure(HeapRegion *_cur) : cur(_cur), young_regions(0),
+ free_regions(0), unclean_regions(0),
+ hum_regions(0),
+ count(0), unaccounted(0),
+ cur_alloc(0), total_free(0)
+ {}
+ bool doHeapRegion(HeapRegion *r) {
+ count++;
+ if (r->is_on_free_list()) free_regions++;
+ else if (r->is_on_unclean_list()) unclean_regions++;
+ else if (r->isHumongous()) hum_regions++;
+ else if (r->is_young()) young_regions++;
+ else if (r == cur) cur_alloc++;
+ else unaccounted++;
+ return false;
+ }
+ void print() {
+ total_free = free_regions + unclean_regions;
+ gclog_or_tty->print("%d regions\n", count);
+ gclog_or_tty->print("%d free: free_list = %d unclean = %d\n",
+ total_free, free_regions, unclean_regions);
+ gclog_or_tty->print("%d humongous %d young\n",
+ hum_regions, young_regions);
+ gclog_or_tty->print("%d cur_alloc\n", cur_alloc);
+ gclog_or_tty->print("UHOH unaccounted = %d\n", unaccounted);
+ }
+};
+
+void G1CollectedHeap::print_region_counts() {
+ SortHeapRegionClosure sc(_cur_alloc_region);
+ PrintHeapRegionClosure cl;
+ heap_region_iterate(&cl);
+ heap_region_iterate(&sc);
+ sc.print();
+ print_region_accounting_info();
+};
+
+bool G1CollectedHeap::regions_accounted_for() {
+ // TODO: regions accounting for young/survivor/tenured
+ return true;
+}
+
+bool G1CollectedHeap::print_region_accounting_info() {
+ gclog_or_tty->print_cr("P regions: %d.", G1NumPopularRegions);
+ gclog_or_tty->print_cr("Free regions: %d (count: %d count list %d) (clean: %d unclean: %d).",
+ free_regions(),
+ count_free_regions(), count_free_regions_list(),
+ _free_region_list_size, _unclean_region_list.sz());
+ gclog_or_tty->print_cr("cur_alloc: %d.",
+ (_cur_alloc_region == NULL ? 0 : 1));
+ gclog_or_tty->print_cr("H regions: %d.", _num_humongous_regions);
+
+ // TODO: check regions accounting for young/survivor/tenured
+ return true;
+}
+
+bool G1CollectedHeap::is_in_closed_subset(const void* p) const {
+ HeapRegion* hr = heap_region_containing(p);
+ if (hr == NULL) {
+ return is_in_permanent(p);
+ } else {
+ return hr->is_in(p);
+ }
+}
+#endif // PRODUCT
+
+void G1CollectedHeap::g1_unimplemented() {
+ // Unimplemented();
+}
+
+
+// Local Variables: ***
+// c-indentation-style: gnu ***
+// End: ***
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
new file mode 100644
index 00000000000..e83b1a7de1c
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp
@@ -0,0 +1,1203 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// A "G1CollectedHeap" is an implementation of a java heap for HotSpot.
+// It uses the "Garbage First" heap organization and algorithm, which
+// may combine concurrent marking with parallel, incremental compaction of
+// heap subsets that will yield large amounts of garbage.
+
+class HeapRegion;
+class HeapRegionSeq;
+class HeapRegionList;
+class PermanentGenerationSpec;
+class GenerationSpec;
+class OopsInHeapRegionClosure;
+class G1ScanHeapEvacClosure;
+class ObjectClosure;
+class SpaceClosure;
+class CompactibleSpaceClosure;
+class Space;
+class G1CollectorPolicy;
+class GenRemSet;
+class G1RemSet;
+class HeapRegionRemSetIterator;
+class ConcurrentMark;
+class ConcurrentMarkThread;
+class ConcurrentG1Refine;
+class ConcurrentZFThread;
+
+// If want to accumulate detailed statistics on work queues
+// turn this on.
+#define G1_DETAILED_STATS 0
+
+#if G1_DETAILED_STATS
+# define IF_G1_DETAILED_STATS(code) code
+#else
+# define IF_G1_DETAILED_STATS(code)
+#endif
+
+typedef GenericTaskQueue RefToScanQueue;
+typedef GenericTaskQueueSet RefToScanQueueSet;
+
+enum G1GCThreadGroups {
+ G1CRGroup = 0,
+ G1ZFGroup = 1,
+ G1CMGroup = 2,
+ G1CLGroup = 3
+};
+
+enum GCAllocPurpose {
+ GCAllocForTenured,
+ GCAllocForSurvived,
+ GCAllocPurposeCount
+};
+
+class YoungList : public CHeapObj {
+private:
+ G1CollectedHeap* _g1h;
+
+ HeapRegion* _head;
+
+ HeapRegion* _scan_only_head;
+ HeapRegion* _scan_only_tail;
+ size_t _length;
+ size_t _scan_only_length;
+
+ size_t _last_sampled_rs_lengths;
+ size_t _sampled_rs_lengths;
+ HeapRegion* _curr;
+ HeapRegion* _curr_scan_only;
+
+ HeapRegion* _survivor_head;
+ HeapRegion* _survivors_tail;
+ size_t _survivor_length;
+
+ void empty_list(HeapRegion* list);
+
+public:
+ YoungList(G1CollectedHeap* g1h);
+
+ void push_region(HeapRegion* hr);
+ void add_survivor_region(HeapRegion* hr);
+ HeapRegion* pop_region();
+ void empty_list();
+ bool is_empty() { return _length == 0; }
+ size_t length() { return _length; }
+ size_t scan_only_length() { return _scan_only_length; }
+
+ void rs_length_sampling_init();
+ bool rs_length_sampling_more();
+ void rs_length_sampling_next();
+
+ void reset_sampled_info() {
+ _last_sampled_rs_lengths = 0;
+ }
+ size_t sampled_rs_lengths() { return _last_sampled_rs_lengths; }
+
+ // for development purposes
+ void reset_auxilary_lists();
+ HeapRegion* first_region() { return _head; }
+ HeapRegion* first_scan_only_region() { return _scan_only_head; }
+ HeapRegion* first_survivor_region() { return _survivor_head; }
+ HeapRegion* par_get_next_scan_only_region() {
+ MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+ HeapRegion* ret = _curr_scan_only;
+ if (ret != NULL)
+ _curr_scan_only = ret->get_next_young_region();
+ return ret;
+ }
+
+ // debugging
+ bool check_list_well_formed();
+ bool check_list_empty(bool ignore_scan_only_list,
+ bool check_sample = true);
+ void print();
+};
+
+class RefineCardTableEntryClosure;
+class G1CollectedHeap : public SharedHeap {
+ friend class VM_G1CollectForAllocation;
+ friend class VM_GenCollectForPermanentAllocation;
+ friend class VM_G1CollectFull;
+ friend class VM_G1IncCollectionPause;
+ friend class VM_G1PopRegionCollectionPause;
+ friend class VMStructs;
+
+ // Closures used in implementation.
+ friend class G1ParCopyHelper;
+ friend class G1IsAliveClosure;
+ friend class G1EvacuateFollowersClosure;
+ friend class G1ParScanThreadState;
+ friend class G1ParScanClosureSuper;
+ friend class G1ParEvacuateFollowersClosure;
+ friend class G1ParTask;
+ friend class G1FreeGarbageRegionClosure;
+ friend class RefineCardTableEntryClosure;
+ friend class G1PrepareCompactClosure;
+ friend class RegionSorter;
+ friend class CountRCClosure;
+ friend class EvacPopObjClosure;
+
+ // Other related classes.
+ friend class G1MarkSweep;
+
+private:
+ enum SomePrivateConstants {
+ VeryLargeInBytes = HeapRegion::GrainBytes/2,
+ VeryLargeInWords = VeryLargeInBytes/HeapWordSize,
+ MinHeapDeltaBytes = 10 * HeapRegion::GrainBytes, // FIXME
+ NumAPIs = HeapRegion::MaxAge
+ };
+
+
+ // The one and only G1CollectedHeap, so static functions can find it.
+ static G1CollectedHeap* _g1h;
+
+ // Storage for the G1 heap (excludes the permanent generation).
+ VirtualSpace _g1_storage;
+ MemRegion _g1_reserved;
+
+ // The part of _g1_storage that is currently committed.
+ MemRegion _g1_committed;
+
+ // The maximum part of _g1_storage that has ever been committed.
+ MemRegion _g1_max_committed;
+
+ // The number of regions that are completely free.
+ size_t _free_regions;
+
+ // The number of regions we could create by expansion.
+ size_t _expansion_regions;
+
+ // Return the number of free regions in the heap (by direct counting.)
+ size_t count_free_regions();
+ // Return the number of free regions on the free and unclean lists.
+ size_t count_free_regions_list();
+
+ // The block offset table for the G1 heap.
+ G1BlockOffsetSharedArray* _bot_shared;
+
+ // Move all of the regions off the free lists, then rebuild those free
+ // lists, before and after full GC.
+ void tear_down_region_lists();
+ void rebuild_region_lists();
+ // This sets all non-empty regions to need zero-fill (which they will if
+ // they are empty after full collection.)
+ void set_used_regions_to_need_zero_fill();
+
+ // The sequence of all heap regions in the heap.
+ HeapRegionSeq* _hrs;
+
+ // The region from which normal-sized objects are currently being
+ // allocated. May be NULL.
+ HeapRegion* _cur_alloc_region;
+
+ // Postcondition: cur_alloc_region == NULL.
+ void abandon_cur_alloc_region();
+
+ // The to-space memory regions into which objects are being copied during
+ // a GC.
+ HeapRegion* _gc_alloc_regions[GCAllocPurposeCount];
+ uint _gc_alloc_region_counts[GCAllocPurposeCount];
+
+ // A list of the regions that have been set to be alloc regions in the
+ // current collection.
+ HeapRegion* _gc_alloc_region_list;
+
+ // When called by par thread, require par_alloc_during_gc_lock() to be held.
+ void push_gc_alloc_region(HeapRegion* hr);
+
+ // This should only be called single-threaded. Undeclares all GC alloc
+ // regions.
+ void forget_alloc_region_list();
+
+ // Should be used to set an alloc region, because there's other
+ // associated bookkeeping.
+ void set_gc_alloc_region(int purpose, HeapRegion* r);
+
+ // Check well-formedness of alloc region list.
+ bool check_gc_alloc_regions();
+
+ // Outside of GC pauses, the number of bytes used in all regions other
+ // than the current allocation region.
+ size_t _summary_bytes_used;
+
+ // Summary information about popular objects; method to print it.
+ NumberSeq _pop_obj_rc_at_copy;
+ void print_popularity_summary_info() const;
+
+ volatile unsigned _gc_time_stamp;
+
+ size_t* _surviving_young_words;
+
+ void setup_surviving_young_words();
+ void update_surviving_young_words(size_t* surv_young_words);
+ void cleanup_surviving_young_words();
+
+protected:
+
+ // Returns "true" iff none of the gc alloc regions have any allocations
+ // since the last call to "save_marks".
+ bool all_alloc_regions_no_allocs_since_save_marks();
+ // Calls "note_end_of_copying on all gc alloc_regions.
+ void all_alloc_regions_note_end_of_copying();
+
+ // The number of regions allocated to hold humongous objects.
+ int _num_humongous_regions;
+ YoungList* _young_list;
+
+ // The current policy object for the collector.
+ G1CollectorPolicy* _g1_policy;
+
+ // Parallel allocation lock to protect the current allocation region.
+ Mutex _par_alloc_during_gc_lock;
+ Mutex* par_alloc_during_gc_lock() { return &_par_alloc_during_gc_lock; }
+
+ // If possible/desirable, allocate a new HeapRegion for normal object
+ // allocation sufficient for an allocation of the given "word_size".
+ // If "do_expand" is true, will attempt to expand the heap if necessary
+ // to to satisfy the request. If "zero_filled" is true, requires a
+ // zero-filled region.
+ // (Returning NULL will trigger a GC.)
+ virtual HeapRegion* newAllocRegion_work(size_t word_size,
+ bool do_expand,
+ bool zero_filled);
+
+ virtual HeapRegion* newAllocRegion(size_t word_size,
+ bool zero_filled = true) {
+ return newAllocRegion_work(word_size, false, zero_filled);
+ }
+ virtual HeapRegion* newAllocRegionWithExpansion(int purpose,
+ size_t word_size,
+ bool zero_filled = true);
+
+ // Attempt to allocate an object of the given (very large) "word_size".
+ // Returns "NULL" on failure.
+ virtual HeapWord* humongousObjAllocate(size_t word_size);
+
+ // If possible, allocate a block of the given word_size, else return "NULL".
+ // Returning NULL will trigger GC or heap expansion.
+ // These two methods have rather awkward pre- and
+ // post-conditions. If they are called outside a safepoint, then
+ // they assume that the caller is holding the heap lock. Upon return
+ // they release the heap lock, if they are returning a non-NULL
+ // value. attempt_allocation_slow() also dirties the cards of a
+ // newly-allocated young region after it releases the heap
+ // lock. This change in interface was the neatest way to achieve
+ // this card dirtying without affecting mem_allocate(), which is a
+ // more frequently called method. We tried two or three different
+ // approaches, but they were even more hacky.
+ HeapWord* attempt_allocation(size_t word_size,
+ bool permit_collection_pause = true);
+
+ HeapWord* attempt_allocation_slow(size_t word_size,
+ bool permit_collection_pause = true);
+
+ // Allocate blocks during garbage collection. Will ensure an
+ // allocation region, either by picking one or expanding the
+ // heap, and then allocate a block of the given size. The block
+ // may not be a humongous - it must fit into a single heap region.
+ HeapWord* allocate_during_gc(GCAllocPurpose purpose, size_t word_size);
+ HeapWord* par_allocate_during_gc(GCAllocPurpose purpose, size_t word_size);
+
+ HeapWord* allocate_during_gc_slow(GCAllocPurpose purpose,
+ HeapRegion* alloc_region,
+ bool par,
+ size_t word_size);
+
+ // Ensure that no further allocations can happen in "r", bearing in mind
+ // that parallel threads might be attempting allocations.
+ void par_allocate_remaining_space(HeapRegion* r);
+
+ // Helper function for two callbacks below.
+ // "full", if true, indicates that the GC is for a System.gc() request,
+ // and should collect the entire heap. If "clear_all_soft_refs" is true,
+ // all soft references are cleared during the GC. If "full" is false,
+ // "word_size" describes the allocation that the GC should
+ // attempt (at least) to satisfy.
+ void do_collection(bool full, bool clear_all_soft_refs,
+ size_t word_size);
+
+ // Callback from VM_G1CollectFull operation.
+ // Perform a full collection.
+ void do_full_collection(bool clear_all_soft_refs);
+
+ // Resize the heap if necessary after a full collection. If this is
+ // after a collect-for allocation, "word_size" is the allocation size,
+ // and will be considered part of the used portion of the heap.
+ void resize_if_necessary_after_full_collection(size_t word_size);
+
+ // Callback from VM_G1CollectForAllocation operation.
+ // This function does everything necessary/possible to satisfy a
+ // failed allocation request (including collection, expansion, etc.)
+ HeapWord* satisfy_failed_allocation(size_t word_size);
+
+ // Attempting to expand the heap sufficiently
+ // to support an allocation of the given "word_size". If
+ // successful, perform the allocation and return the address of the
+ // allocated block, or else "NULL".
+ virtual HeapWord* expand_and_allocate(size_t word_size);
+
+public:
+ // Expand the garbage-first heap by at least the given size (in bytes!).
+ // (Rounds up to a HeapRegion boundary.)
+ virtual void expand(size_t expand_bytes);
+
+ // Do anything common to GC's.
+ virtual void gc_prologue(bool full);
+ virtual void gc_epilogue(bool full);
+
+protected:
+
+ // Shrink the garbage-first heap by at most the given size (in bytes!).
+ // (Rounds down to a HeapRegion boundary.)
+ virtual void shrink(size_t expand_bytes);
+ void shrink_helper(size_t expand_bytes);
+
+ // Do an incremental collection: identify a collection set, and evacuate
+ // its live objects elsewhere.
+ virtual void do_collection_pause();
+
+ // The guts of the incremental collection pause, executed by the vm
+ // thread. If "popular_region" is non-NULL, this pause should evacuate
+ // this single region whose remembered set has gotten large, moving
+ // any popular objects to one of the popular regions.
+ virtual void do_collection_pause_at_safepoint(HeapRegion* popular_region);
+
+ // Actually do the work of evacuating the collection set.
+ virtual void evacuate_collection_set();
+
+ // If this is an appropriate right time, do a collection pause.
+ // The "word_size" argument, if non-zero, indicates the size of an
+ // allocation request that is prompting this query.
+ void do_collection_pause_if_appropriate(size_t word_size);
+
+ // The g1 remembered set of the heap.
+ G1RemSet* _g1_rem_set;
+ // And it's mod ref barrier set, used to track updates for the above.
+ ModRefBarrierSet* _mr_bs;
+
+ // The Heap Region Rem Set Iterator.
+ HeapRegionRemSetIterator** _rem_set_iterator;
+
+ // The closure used to refine a single card.
+ RefineCardTableEntryClosure* _refine_cte_cl;
+
+ // A function to check the consistency of dirty card logs.
+ void check_ct_logs_at_safepoint();
+
+ // After a collection pause, make the regions in the CS into free
+ // regions.
+ void free_collection_set(HeapRegion* cs_head);
+
+ // Applies "scan_non_heap_roots" to roots outside the heap,
+ // "scan_rs" to roots inside the heap (having done "set_region" to
+ // indicate the region in which the root resides), and does "scan_perm"
+ // (setting the generation to the perm generation.) If "scan_rs" is
+ // NULL, then this step is skipped. The "worker_i"
+ // param is for use with parallel roots processing, and should be
+ // the "i" of the calling parallel worker thread's work(i) function.
+ // In the sequential case this param will be ignored.
+ void g1_process_strong_roots(bool collecting_perm_gen,
+ SharedHeap::ScanningOption so,
+ OopClosure* scan_non_heap_roots,
+ OopsInHeapRegionClosure* scan_rs,
+ OopsInHeapRegionClosure* scan_so,
+ OopsInGenClosure* scan_perm,
+ int worker_i);
+
+ void scan_scan_only_set(OopsInHeapRegionClosure* oc,
+ int worker_i);
+ void scan_scan_only_region(HeapRegion* hr,
+ OopsInHeapRegionClosure* oc,
+ int worker_i);
+
+ // Apply "blk" to all the weak roots of the system. These include
+ // JNI weak roots, the code cache, system dictionary, symbol table,
+ // string table, and referents of reachable weak refs.
+ void g1_process_weak_roots(OopClosure* root_closure,
+ OopClosure* non_root_closure);
+
+ // Invoke "save_marks" on all heap regions.
+ void save_marks();
+
+ // Free a heap region.
+ void free_region(HeapRegion* hr);
+ // A component of "free_region", exposed for 'batching'.
+ // All the params after "hr" are out params: the used bytes of the freed
+ // region(s), the number of H regions cleared, the number of regions
+ // freed, and pointers to the head and tail of a list of freed contig
+ // regions, linked throught the "next_on_unclean_list" field.
+ void free_region_work(HeapRegion* hr,
+ size_t& pre_used,
+ size_t& cleared_h,
+ size_t& freed_regions,
+ UncleanRegionList* list,
+ bool par = false);
+
+
+ // The concurrent marker (and the thread it runs in.)
+ ConcurrentMark* _cm;
+ ConcurrentMarkThread* _cmThread;
+ bool _mark_in_progress;
+
+ // The concurrent refiner.
+ ConcurrentG1Refine* _cg1r;
+
+ // The concurrent zero-fill thread.
+ ConcurrentZFThread* _czft;
+
+ // The parallel task queues
+ RefToScanQueueSet *_task_queues;
+
+ // True iff a evacuation has failed in the current collection.
+ bool _evacuation_failed;
+
+ // Set the attribute indicating whether evacuation has failed in the
+ // current collection.
+ void set_evacuation_failed(bool b) { _evacuation_failed = b; }
+
+ // Failed evacuations cause some logical from-space objects to have
+ // forwarding pointers to themselves. Reset them.
+ void remove_self_forwarding_pointers();
+
+ // When one is non-null, so is the other. Together, they each pair is
+ // an object with a preserved mark, and its mark value.
+ GrowableArray* _objs_with_preserved_marks;
+ GrowableArray* _preserved_marks_of_objs;
+
+ // Preserve the mark of "obj", if necessary, in preparation for its mark
+ // word being overwritten with a self-forwarding-pointer.
+ void preserve_mark_if_necessary(oop obj, markOop m);
+
+ // The stack of evac-failure objects left to be scanned.
+ GrowableArray* _evac_failure_scan_stack;
+ // The closure to apply to evac-failure objects.
+
+ OopsInHeapRegionClosure* _evac_failure_closure;
+ // Set the field above.
+ void
+ set_evac_failure_closure(OopsInHeapRegionClosure* evac_failure_closure) {
+ _evac_failure_closure = evac_failure_closure;
+ }
+
+ // Push "obj" on the scan stack.
+ void push_on_evac_failure_scan_stack(oop obj);
+ // Process scan stack entries until the stack is empty.
+ void drain_evac_failure_scan_stack();
+ // True iff an invocation of "drain_scan_stack" is in progress; to
+ // prevent unnecessary recursion.
+ bool _drain_in_progress;
+
+ // Do any necessary initialization for evacuation-failure handling.
+ // "cl" is the closure that will be used to process evac-failure
+ // objects.
+ void init_for_evac_failure(OopsInHeapRegionClosure* cl);
+ // Do any necessary cleanup for evacuation-failure handling data
+ // structures.
+ void finalize_for_evac_failure();
+
+ // An attempt to evacuate "obj" has failed; take necessary steps.
+ void handle_evacuation_failure(oop obj);
+ oop handle_evacuation_failure_par(OopsInHeapRegionClosure* cl, oop obj);
+ void handle_evacuation_failure_common(oop obj, markOop m);
+
+
+ // Ensure that the relevant gc_alloc regions are set.
+ void get_gc_alloc_regions();
+ // We're done with GC alloc regions; release them, as appropriate.
+ void release_gc_alloc_regions();
+
+ // ("Weak") Reference processing support
+ ReferenceProcessor* _ref_processor;
+
+ enum G1H_process_strong_roots_tasks {
+ G1H_PS_mark_stack_oops_do,
+ G1H_PS_refProcessor_oops_do,
+ // Leave this one last.
+ G1H_PS_NumElements
+ };
+
+ SubTasksDone* _process_strong_tasks;
+
+ // Allocate space to hold a popular object. Result is guaranteed below
+ // "popular_object_boundary()". Note: CURRENTLY halts the system if we
+ // run out of space to hold popular objects.
+ HeapWord* allocate_popular_object(size_t word_size);
+
+ // The boundary between popular and non-popular objects.
+ HeapWord* _popular_object_boundary;
+
+ HeapRegionList* _popular_regions_to_be_evacuated;
+
+ // Compute which objects in "single_region" are popular. If any are,
+ // evacuate them to a popular region, leaving behind forwarding pointers,
+ // and select "popular_region" as the single collection set region.
+ // Otherwise, leave the collection set null.
+ void popularity_pause_preamble(HeapRegion* populer_region);
+
+ // Compute which objects in "single_region" are popular, and evacuate
+ // them to a popular region, leaving behind forwarding pointers.
+ // Returns "true" if at least one popular object is discovered and
+ // evacuated. In any case, "*max_rc" is set to the maximum reference
+ // count of an object in the region.
+ bool compute_reference_counts_and_evac_popular(HeapRegion* populer_region,
+ size_t* max_rc);
+ // Subroutines used in the above.
+ bool _rc_region_above;
+ size_t _rc_region_diff;
+ jint* obj_rc_addr(oop obj) {
+ uintptr_t obj_addr = (uintptr_t)obj;
+ if (_rc_region_above) {
+ jint* res = (jint*)(obj_addr + _rc_region_diff);
+ assert((uintptr_t)res > obj_addr, "RC region is above.");
+ return res;
+ } else {
+ jint* res = (jint*)(obj_addr - _rc_region_diff);
+ assert((uintptr_t)res < obj_addr, "RC region is below.");
+ return res;
+ }
+ }
+ jint obj_rc(oop obj) {
+ return *obj_rc_addr(obj);
+ }
+ void inc_obj_rc(oop obj) {
+ (*obj_rc_addr(obj))++;
+ }
+ void atomic_inc_obj_rc(oop obj);
+
+
+ // Number of popular objects and bytes (latter is cheaper!).
+ size_t pop_object_used_objs();
+ size_t pop_object_used_bytes();
+
+ // Index of the popular region in which allocation is currently being
+ // done.
+ int _cur_pop_hr_index;
+
+ // List of regions which require zero filling.
+ UncleanRegionList _unclean_region_list;
+ bool _unclean_regions_coming;
+
+ bool check_age_cohort_well_formed_work(int a, HeapRegion* hr);
+
+public:
+ void set_refine_cte_cl_concurrency(bool concurrent);
+
+ RefToScanQueue *task_queue(int i);
+
+ // Create a G1CollectedHeap with the specified policy.
+ // Must call the initialize method afterwards.
+ // May not return if something goes wrong.
+ G1CollectedHeap(G1CollectorPolicy* policy);
+
+ // Initialize the G1CollectedHeap to have the initial and
+ // maximum sizes, permanent generation, and remembered and barrier sets
+ // specified by the policy object.
+ jint initialize();
+
+ void ref_processing_init();
+
+ void set_par_threads(int t) {
+ SharedHeap::set_par_threads(t);
+ _process_strong_tasks->set_par_threads(t);
+ }
+
+ virtual CollectedHeap::Name kind() const {
+ return CollectedHeap::G1CollectedHeap;
+ }
+
+ // The current policy object for the collector.
+ G1CollectorPolicy* g1_policy() const { return _g1_policy; }
+
+ // Adaptive size policy. No such thing for g1.
+ virtual AdaptiveSizePolicy* size_policy() { return NULL; }
+
+ // The rem set and barrier set.
+ G1RemSet* g1_rem_set() const { return _g1_rem_set; }
+ ModRefBarrierSet* mr_bs() const { return _mr_bs; }
+
+ // The rem set iterator.
+ HeapRegionRemSetIterator* rem_set_iterator(int i) {
+ return _rem_set_iterator[i];
+ }
+
+ HeapRegionRemSetIterator* rem_set_iterator() {
+ return _rem_set_iterator[0];
+ }
+
+ unsigned get_gc_time_stamp() {
+ return _gc_time_stamp;
+ }
+
+ void reset_gc_time_stamp() {
+ _gc_time_stamp = 0;
+ OrderAccess::fence();
+ }
+
+ void increment_gc_time_stamp() {
+ ++_gc_time_stamp;
+ OrderAccess::fence();
+ }
+
+ void iterate_dirty_card_closure(bool concurrent, int worker_i);
+
+ // The shared block offset table array.
+ G1BlockOffsetSharedArray* bot_shared() const { return _bot_shared; }
+
+ // Reference Processing accessor
+ ReferenceProcessor* ref_processor() { return _ref_processor; }
+
+ // Reserved (g1 only; super method includes perm), capacity and the used
+ // portion in bytes.
+ size_t g1_reserved_obj_bytes() { return _g1_reserved.byte_size(); }
+ virtual size_t capacity() const;
+ virtual size_t used() const;
+ size_t recalculate_used() const;
+#ifndef PRODUCT
+ size_t recalculate_used_regions() const;
+#endif // PRODUCT
+
+ // These virtual functions do the actual allocation.
+ virtual HeapWord* mem_allocate(size_t word_size,
+ bool is_noref,
+ bool is_tlab,
+ bool* gc_overhead_limit_was_exceeded);
+
+ // Some heaps may offer a contiguous region for shared non-blocking
+ // allocation, via inlined code (by exporting the address of the top and
+ // end fields defining the extent of the contiguous allocation region.)
+ // But G1CollectedHeap doesn't yet support this.
+
+ // Return an estimate of the maximum allocation that could be performed
+ // without triggering any collection or expansion activity. In a
+ // generational collector, for example, this is probably the largest
+ // allocation that could be supported (without expansion) in the youngest
+ // generation. It is "unsafe" because no locks are taken; the result
+ // should be treated as an approximation, not a guarantee, for use in
+ // heuristic resizing decisions.
+ virtual size_t unsafe_max_alloc();
+
+ virtual bool is_maximal_no_gc() const {
+ return _g1_storage.uncommitted_size() == 0;
+ }
+
+ // The total number of regions in the heap.
+ size_t n_regions();
+
+ // The number of regions that are completely free.
+ size_t max_regions();
+
+ // The number of regions that are completely free.
+ size_t free_regions();
+
+ // The number of regions that are not completely free.
+ size_t used_regions() { return n_regions() - free_regions(); }
+
+ // True iff the ZF thread should run.
+ bool should_zf();
+
+ // The number of regions available for "regular" expansion.
+ size_t expansion_regions() { return _expansion_regions; }
+
+#ifndef PRODUCT
+ bool regions_accounted_for();
+ bool print_region_accounting_info();
+ void print_region_counts();
+#endif
+
+ HeapRegion* alloc_region_from_unclean_list(bool zero_filled);
+ HeapRegion* alloc_region_from_unclean_list_locked(bool zero_filled);
+
+ void put_region_on_unclean_list(HeapRegion* r);
+ void put_region_on_unclean_list_locked(HeapRegion* r);
+
+ void prepend_region_list_on_unclean_list(UncleanRegionList* list);
+ void prepend_region_list_on_unclean_list_locked(UncleanRegionList* list);
+
+ void set_unclean_regions_coming(bool b);
+ void set_unclean_regions_coming_locked(bool b);
+ // Wait for cleanup to be complete.
+ void wait_for_cleanup_complete();
+ // Like above, but assumes that the calling thread owns the Heap_lock.
+ void wait_for_cleanup_complete_locked();
+
+ // Return the head of the unclean list.
+ HeapRegion* peek_unclean_region_list_locked();
+ // Remove and return the head of the unclean list.
+ HeapRegion* pop_unclean_region_list_locked();
+
+ // List of regions which are zero filled and ready for allocation.
+ HeapRegion* _free_region_list;
+ // Number of elements on the free list.
+ size_t _free_region_list_size;
+
+ // If the head of the unclean list is ZeroFilled, move it to the free
+ // list.
+ bool move_cleaned_region_to_free_list_locked();
+ bool move_cleaned_region_to_free_list();
+
+ void put_free_region_on_list_locked(HeapRegion* r);
+ void put_free_region_on_list(HeapRegion* r);
+
+ // Remove and return the head element of the free list.
+ HeapRegion* pop_free_region_list_locked();
+
+ // If "zero_filled" is true, we first try the free list, then we try the
+ // unclean list, zero-filling the result. If "zero_filled" is false, we
+ // first try the unclean list, then the zero-filled list.
+ HeapRegion* alloc_free_region_from_lists(bool zero_filled);
+
+ // Verify the integrity of the region lists.
+ void remove_allocated_regions_from_lists();
+ bool verify_region_lists();
+ bool verify_region_lists_locked();
+ size_t unclean_region_list_length();
+ size_t free_region_list_length();
+
+ // Perform a collection of the heap; intended for use in implementing
+ // "System.gc". This probably implies as full a collection as the
+ // "CollectedHeap" supports.
+ virtual void collect(GCCause::Cause cause);
+
+ // The same as above but assume that the caller holds the Heap_lock.
+ void collect_locked(GCCause::Cause cause);
+
+ // This interface assumes that it's being called by the
+ // vm thread. It collects the heap assuming that the
+ // heap lock is already held and that we are executing in
+ // the context of the vm thread.
+ virtual void collect_as_vm_thread(GCCause::Cause cause);
+
+ // True iff a evacuation has failed in the most-recent collection.
+ bool evacuation_failed() { return _evacuation_failed; }
+
+ // Free a region if it is totally full of garbage. Returns the number of
+ // bytes freed (0 ==> didn't free it).
+ size_t free_region_if_totally_empty(HeapRegion *hr);
+ void free_region_if_totally_empty_work(HeapRegion *hr,
+ size_t& pre_used,
+ size_t& cleared_h_regions,
+ size_t& freed_regions,
+ UncleanRegionList* list,
+ bool par = false);
+
+ // If we've done free region work that yields the given changes, update
+ // the relevant global variables.
+ void finish_free_region_work(size_t pre_used,
+ size_t cleared_h_regions,
+ size_t freed_regions,
+ UncleanRegionList* list);
+
+
+ // Returns "TRUE" iff "p" points into the allocated area of the heap.
+ virtual bool is_in(const void* p) const;
+
+ // Return "TRUE" iff the given object address is within the collection
+ // set.
+ inline bool obj_in_cs(oop obj);
+
+ // Return "TRUE" iff the given object address is in the reserved
+ // region of g1 (excluding the permanent generation).
+ bool is_in_g1_reserved(const void* p) const {
+ return _g1_reserved.contains(p);
+ }
+
+ // Returns a MemRegion that corresponds to the space that has been
+ // committed in the heap
+ MemRegion g1_committed() {
+ return _g1_committed;
+ }
+
+ NOT_PRODUCT( bool is_in_closed_subset(const void* p) const; )
+
+ // Dirty card table entries covering a list of young regions.
+ void dirtyCardsForYoungRegions(CardTableModRefBS* ct_bs, HeapRegion* list);
+
+ // This resets the card table to all zeros. It is used after
+ // a collection pause which used the card table to claim cards.
+ void cleanUpCardTable();
+
+ // Iteration functions.
+
+ // Iterate over all the ref-containing fields of all objects, calling
+ // "cl.do_oop" on each.
+ virtual void oop_iterate(OopClosure* cl);
+
+ // Same as above, restricted to a memory region.
+ virtual void oop_iterate(MemRegion mr, OopClosure* cl);
+
+ // Iterate over all objects, calling "cl.do_object" on each.
+ virtual void object_iterate(ObjectClosure* cl);
+
+ // Iterate over all objects allocated since the last collection, calling
+ // "cl.do_object" on each. The heap must have been initialized properly
+ // to support this function, or else this call will fail.
+ virtual void object_iterate_since_last_GC(ObjectClosure* cl);
+
+ // Iterate over all spaces in use in the heap, in ascending address order.
+ virtual void space_iterate(SpaceClosure* cl);
+
+ // Iterate over heap regions, in address order, terminating the
+ // iteration early if the "doHeapRegion" method returns "true".
+ void heap_region_iterate(HeapRegionClosure* blk);
+
+ // Iterate over heap regions starting with r (or the first region if "r"
+ // is NULL), in address order, terminating early if the "doHeapRegion"
+ // method returns "true".
+ void heap_region_iterate_from(HeapRegion* r, HeapRegionClosure* blk);
+
+ // As above but starting from the region at index idx.
+ void heap_region_iterate_from(int idx, HeapRegionClosure* blk);
+
+ HeapRegion* region_at(size_t idx);
+
+ // Divide the heap region sequence into "chunks" of some size (the number
+ // of regions divided by the number of parallel threads times some
+ // overpartition factor, currently 4). Assumes that this will be called
+ // in parallel by ParallelGCThreads worker threads with discinct worker
+ // ids in the range [0..max(ParallelGCThreads-1, 1)], that all parallel
+ // calls will use the same "claim_value", and that that claim value is
+ // different from the claim_value of any heap region before the start of
+ // the iteration. Applies "blk->doHeapRegion" to each of the regions, by
+ // attempting to claim the first region in each chunk, and, if
+ // successful, applying the closure to each region in the chunk (and
+ // setting the claim value of the second and subsequent regions of the
+ // chunk.) For now requires that "doHeapRegion" always returns "false",
+ // i.e., that a closure never attempt to abort a traversal.
+ void heap_region_par_iterate_chunked(HeapRegionClosure* blk,
+ int worker,
+ jint claim_value);
+
+ // It resets all the region claim values to the default.
+ void reset_heap_region_claim_values();
+
+#ifdef ASSERT
+ bool check_heap_region_claim_values(jint claim_value);
+#endif // ASSERT
+
+ // Iterate over the regions (if any) in the current collection set.
+ void collection_set_iterate(HeapRegionClosure* blk);
+
+ // As above but starting from region r
+ void collection_set_iterate_from(HeapRegion* r, HeapRegionClosure *blk);
+
+ // Returns the first (lowest address) compactible space in the heap.
+ virtual CompactibleSpace* first_compactible_space();
+
+ // A CollectedHeap will contain some number of spaces. This finds the
+ // space containing a given address, or else returns NULL.
+ virtual Space* space_containing(const void* addr) const;
+
+ // A G1CollectedHeap will contain some number of heap regions. This
+ // finds the region containing a given address, or else returns NULL.
+ HeapRegion* heap_region_containing(const void* addr) const;
+
+ // Like the above, but requires "addr" to be in the heap (to avoid a
+ // null-check), and unlike the above, may return an continuing humongous
+ // region.
+ HeapRegion* heap_region_containing_raw(const void* addr) const;
+
+ // A CollectedHeap is divided into a dense sequence of "blocks"; that is,
+ // each address in the (reserved) heap is a member of exactly
+ // one block. The defining characteristic of a block is that it is
+ // possible to find its size, and thus to progress forward to the next
+ // block. (Blocks may be of different sizes.) Thus, blocks may
+ // represent Java objects, or they might be free blocks in a
+ // free-list-based heap (or subheap), as long as the two kinds are
+ // distinguishable and the size of each is determinable.
+
+ // Returns the address of the start of the "block" that contains the
+ // address "addr". We say "blocks" instead of "object" since some heaps
+ // may not pack objects densely; a chunk may either be an object or a
+ // non-object.
+ virtual HeapWord* block_start(const void* addr) const;
+
+ // Requires "addr" to be the start of a chunk, and returns its size.
+ // "addr + size" is required to be the start of a new chunk, or the end
+ // of the active area of the heap.
+ virtual size_t block_size(const HeapWord* addr) const;
+
+ // Requires "addr" to be the start of a block, and returns "TRUE" iff
+ // the block is an object.
+ virtual bool block_is_obj(const HeapWord* addr) const;
+
+ // Does this heap support heap inspection? (+PrintClassHistogram)
+ virtual bool supports_heap_inspection() const { return true; }
+
+ // Section on thread-local allocation buffers (TLABs)
+ // See CollectedHeap for semantics.
+
+ virtual bool supports_tlab_allocation() const;
+ virtual size_t tlab_capacity(Thread* thr) const;
+ virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;
+ virtual HeapWord* allocate_new_tlab(size_t size);
+
+ // Can a compiler initialize a new object without store barriers?
+ // This permission only extends from the creation of a new object
+ // via a TLAB up to the first subsequent safepoint.
+ virtual bool can_elide_tlab_store_barriers() const {
+ // Since G1's TLAB's may, on occasion, come from non-young regions
+ // as well. (Is there a flag controlling that? XXX)
+ return false;
+ }
+
+ // Can a compiler elide a store barrier when it writes
+ // a permanent oop into the heap? Applies when the compiler
+ // is storing x to the heap, where x->is_perm() is true.
+ virtual bool can_elide_permanent_oop_store_barriers() const {
+ // At least until perm gen collection is also G1-ified, at
+ // which point this should return false.
+ return true;
+ }
+
+ virtual bool allocs_are_zero_filled();
+
+ // The boundary between a "large" and "small" array of primitives, in
+ // words.
+ virtual size_t large_typearray_limit();
+
+ // All popular objects are guaranteed to have addresses below this
+ // boundary.
+ HeapWord* popular_object_boundary() {
+ return _popular_object_boundary;
+ }
+
+ // Declare the region as one that should be evacuated because its
+ // remembered set is too large.
+ void schedule_popular_region_evac(HeapRegion* r);
+ // If there is a popular region to evacuate it, remove it from the list
+ // and return it.
+ HeapRegion* popular_region_to_evac();
+ // Evacuate the given popular region.
+ void evac_popular_region(HeapRegion* r);
+
+ // Returns "true" iff the given word_size is "very large".
+ static bool isHumongous(size_t word_size) {
+ return word_size >= VeryLargeInWords;
+ }
+
+ // Update mod union table with the set of dirty cards.
+ void updateModUnion();
+
+ // Set the mod union bits corresponding to the given memRegion. Note
+ // that this is always a safe operation, since it doesn't clear any
+ // bits.
+ void markModUnionRange(MemRegion mr);
+
+ // Records the fact that a marking phase is no longer in progress.
+ void set_marking_complete() {
+ _mark_in_progress = false;
+ }
+ void set_marking_started() {
+ _mark_in_progress = true;
+ }
+ bool mark_in_progress() {
+ return _mark_in_progress;
+ }
+
+ // Print the maximum heap capacity.
+ virtual size_t max_capacity() const;
+
+ virtual jlong millis_since_last_gc();
+
+ // Perform any cleanup actions necessary before allowing a verification.
+ virtual void prepare_for_verify();
+
+ // Perform verification.
+ virtual void verify(bool allow_dirty, bool silent);
+ virtual void print() const;
+ virtual void print_on(outputStream* st) const;
+
+ virtual void print_gc_threads_on(outputStream* st) const;
+ virtual void gc_threads_do(ThreadClosure* tc) const;
+
+ // Override
+ void print_tracing_info() const;
+
+ // If "addr" is a pointer into the (reserved?) heap, returns a positive
+ // number indicating the "arena" within the heap in which "addr" falls.
+ // Or else returns 0.
+ virtual int addr_to_arena_id(void* addr) const;
+
+ // Convenience function to be used in situations where the heap type can be
+ // asserted to be this type.
+ static G1CollectedHeap* heap();
+
+ void empty_young_list();
+ bool should_set_young_locked();
+
+ void set_region_short_lived_locked(HeapRegion* hr);
+ // add appropriate methods for any other surv rate groups
+
+ void young_list_rs_length_sampling_init() {
+ _young_list->rs_length_sampling_init();
+ }
+ bool young_list_rs_length_sampling_more() {
+ return _young_list->rs_length_sampling_more();
+ }
+ void young_list_rs_length_sampling_next() {
+ _young_list->rs_length_sampling_next();
+ }
+ size_t young_list_sampled_rs_lengths() {
+ return _young_list->sampled_rs_lengths();
+ }
+
+ size_t young_list_length() { return _young_list->length(); }
+ size_t young_list_scan_only_length() {
+ return _young_list->scan_only_length(); }
+
+ HeapRegion* pop_region_from_young_list() {
+ return _young_list->pop_region();
+ }
+
+ HeapRegion* young_list_first_region() {
+ return _young_list->first_region();
+ }
+
+ // debugging
+ bool check_young_list_well_formed() {
+ return _young_list->check_list_well_formed();
+ }
+ bool check_young_list_empty(bool ignore_scan_only_list,
+ bool check_sample = true);
+
+ // *** Stuff related to concurrent marking. It's not clear to me that so
+ // many of these need to be public.
+
+ // The functions below are helper functions that a subclass of
+ // "CollectedHeap" can use in the implementation of its virtual
+ // functions.
+ // This performs a concurrent marking of the live objects in a
+ // bitmap off to the side.
+ void doConcurrentMark();
+
+ // This is called from the marksweep collector which then does
+ // a concurrent mark and verifies that the results agree with
+ // the stop the world marking.
+ void checkConcurrentMark();
+ void do_sync_mark();
+
+ bool isMarkedPrev(oop obj) const;
+ bool isMarkedNext(oop obj) const;
+
+ // Determine if an object is dead, given the object and also
+ // the region to which the object belongs. An object is dead
+ // iff a) it was not allocated since the last mark and b) it
+ // is not marked.
+
+ bool is_obj_dead(const oop obj, const HeapRegion* hr) const {
+ return
+ !hr->obj_allocated_since_prev_marking(obj) &&
+ !isMarkedPrev(obj);
+ }
+
+ // This is used when copying an object to survivor space.
+ // If the object is marked live, then we mark the copy live.
+ // If the object is allocated since the start of this mark
+ // cycle, then we mark the copy live.
+ // If the object has been around since the previous mark
+ // phase, and hasn't been marked yet during this phase,
+ // then we don't mark it, we just wait for the
+ // current marking cycle to get to it.
+
+ // This function returns true when an object has been
+ // around since the previous marking and hasn't yet
+ // been marked during this marking.
+
+ bool is_obj_ill(const oop obj, const HeapRegion* hr) const {
+ return
+ !hr->obj_allocated_since_next_marking(obj) &&
+ !isMarkedNext(obj);
+ }
+
+ // Determine if an object is dead, given only the object itself.
+ // This will find the region to which the object belongs and
+ // then call the region version of the same function.
+
+ // Added if it is in permanent gen it isn't dead.
+ // Added if it is NULL it isn't dead.
+
+ bool is_obj_dead(oop obj) {
+ HeapRegion* hr = heap_region_containing(obj);
+ if (hr == NULL) {
+ if (Universe::heap()->is_in_permanent(obj))
+ return false;
+ else if (obj == NULL) return false;
+ else return true;
+ }
+ else return is_obj_dead(obj, hr);
+ }
+
+ bool is_obj_ill(oop obj) {
+ HeapRegion* hr = heap_region_containing(obj);
+ if (hr == NULL) {
+ if (Universe::heap()->is_in_permanent(obj))
+ return false;
+ else if (obj == NULL) return false;
+ else return true;
+ }
+ else return is_obj_ill(obj, hr);
+ }
+
+ // The following is just to alert the verification code
+ // that a full collection has occurred and that the
+ // remembered sets are no longer up to date.
+ bool _full_collection;
+ void set_full_collection() { _full_collection = true;}
+ void clear_full_collection() {_full_collection = false;}
+ bool full_collection() {return _full_collection;}
+
+ ConcurrentMark* concurrent_mark() const { return _cm; }
+ ConcurrentG1Refine* concurrent_g1_refine() const { return _cg1r; }
+
+public:
+ void stop_conc_gc_threads();
+
+ //
+
+ double predict_region_elapsed_time_ms(HeapRegion* hr, bool young);
+ void check_if_region_is_too_expensive(double predicted_time_ms);
+ size_t pending_card_num();
+ size_t max_pending_card_num();
+ size_t cards_scanned();
+
+ //
+
+protected:
+ size_t _max_heap_capacity;
+
+// debug_only(static void check_for_valid_allocation_state();)
+
+public:
+ // Temporary: call to mark things unimplemented for the G1 heap (e.g.,
+ // MemoryService). In productization, we can make this assert false
+ // to catch such places (as well as searching for calls to this...)
+ static void g1_unimplemented();
+
+};
+
+// Local Variables: ***
+// c-indentation-style: gnu ***
+// End: ***
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp
new file mode 100644
index 00000000000..8cafe3d9885
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// Inline functions for G1CollectedHeap
+
+inline HeapRegion*
+G1CollectedHeap::heap_region_containing(const void* addr) const {
+ HeapRegion* hr = _hrs->addr_to_region(addr);
+ // hr can be null if addr in perm_gen
+ if (hr != NULL && hr->continuesHumongous()) {
+ hr = hr->humongous_start_region();
+ }
+ return hr;
+}
+
+inline HeapRegion*
+G1CollectedHeap::heap_region_containing_raw(const void* addr) const {
+ HeapRegion* res = _hrs->addr_to_region(addr);
+ assert(res != NULL, "addr outside of heap?");
+ return res;
+}
+
+inline bool G1CollectedHeap::obj_in_cs(oop obj) {
+ HeapRegion* r = _hrs->addr_to_region(obj);
+ return r != NULL && r->in_collection_set();
+}
+
+inline HeapWord* G1CollectedHeap::attempt_allocation(size_t word_size,
+ bool permit_collection_pause) {
+ HeapWord* res = NULL;
+
+ assert( SafepointSynchronize::is_at_safepoint() ||
+ Heap_lock->owned_by_self(), "pre-condition of the call" );
+
+ if (_cur_alloc_region != NULL) {
+
+ // If this allocation causes a region to become non empty,
+ // then we need to update our free_regions count.
+
+ if (_cur_alloc_region->is_empty()) {
+ res = _cur_alloc_region->allocate(word_size);
+ if (res != NULL)
+ _free_regions--;
+ } else {
+ res = _cur_alloc_region->allocate(word_size);
+ }
+ }
+ if (res != NULL) {
+ if (!SafepointSynchronize::is_at_safepoint()) {
+ assert( Heap_lock->owned_by_self(), "invariant" );
+ Heap_lock->unlock();
+ }
+ return res;
+ }
+ // attempt_allocation_slow will also unlock the heap lock when appropriate.
+ return attempt_allocation_slow(word_size, permit_collection_pause);
+}
+
+inline RefToScanQueue* G1CollectedHeap::task_queue(int i) {
+ return _task_queues->queue(i);
+}
+
+
+inline bool G1CollectedHeap::isMarkedPrev(oop obj) const {
+ return _cm->prevMarkBitMap()->isMarked((HeapWord *)obj);
+}
+
+inline bool G1CollectedHeap::isMarkedNext(oop obj) const {
+ return _cm->nextMarkBitMap()->isMarked((HeapWord *)obj);
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp
new file mode 100644
index 00000000000..a5d0165bb4d
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp
@@ -0,0 +1,3163 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_g1CollectorPolicy.cpp.incl"
+
+#define PREDICTIONS_VERBOSE 0
+
+//
+
+// Different defaults for different number of GC threads
+// They were chosen by running GCOld and SPECjbb on debris with different
+// numbers of GC threads and choosing them based on the results
+
+// all the same
+static double rs_length_diff_defaults[] = {
+ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
+};
+
+static double cost_per_card_ms_defaults[] = {
+ 0.01, 0.005, 0.005, 0.003, 0.003, 0.002, 0.002, 0.0015
+};
+
+static double cost_per_scan_only_region_ms_defaults[] = {
+ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
+};
+
+// all the same
+static double fully_young_cards_per_entry_ratio_defaults[] = {
+ 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0
+};
+
+static double cost_per_entry_ms_defaults[] = {
+ 0.015, 0.01, 0.01, 0.008, 0.008, 0.0055, 0.0055, 0.005
+};
+
+static double cost_per_byte_ms_defaults[] = {
+ 0.00006, 0.00003, 0.00003, 0.000015, 0.000015, 0.00001, 0.00001, 0.000009
+};
+
+// these should be pretty consistent
+static double constant_other_time_ms_defaults[] = {
+ 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0
+};
+
+
+static double young_other_cost_per_region_ms_defaults[] = {
+ 0.3, 0.2, 0.2, 0.15, 0.15, 0.12, 0.12, 0.1
+};
+
+static double non_young_other_cost_per_region_ms_defaults[] = {
+ 1.0, 0.7, 0.7, 0.5, 0.5, 0.42, 0.42, 0.30
+};
+
+//
+
+G1CollectorPolicy::G1CollectorPolicy() :
+ _parallel_gc_threads((ParallelGCThreads > 0) ? ParallelGCThreads : 1),
+ _n_pauses(0),
+ _recent_CH_strong_roots_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_G1_strong_roots_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_evac_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_pause_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_rs_sizes(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_gc_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _all_pause_times_ms(new NumberSeq()),
+ _stop_world_start(0.0),
+ _all_stop_world_times_ms(new NumberSeq()),
+ _all_yield_times_ms(new NumberSeq()),
+
+ _all_mod_union_times_ms(new NumberSeq()),
+
+ _non_pop_summary(new NonPopSummary()),
+ _pop_summary(new PopSummary()),
+ _non_pop_abandoned_summary(new NonPopAbandonedSummary()),
+ _pop_abandoned_summary(new PopAbandonedSummary()),
+
+ _cur_clear_ct_time_ms(0.0),
+
+ _region_num_young(0),
+ _region_num_tenured(0),
+ _prev_region_num_young(0),
+ _prev_region_num_tenured(0),
+
+ _aux_num(10),
+ _all_aux_times_ms(new NumberSeq[_aux_num]),
+ _cur_aux_start_times_ms(new double[_aux_num]),
+ _cur_aux_times_ms(new double[_aux_num]),
+ _cur_aux_times_set(new bool[_aux_num]),
+
+ _pop_compute_rc_start(0.0),
+ _pop_evac_start(0.0),
+
+ _concurrent_mark_init_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _concurrent_mark_remark_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _concurrent_mark_cleanup_times_ms(new TruncatedSeq(NumPrevPausesForHeuristics)),
+
+ //
+
+ _alloc_rate_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _prev_collection_pause_end_ms(0.0),
+ _pending_card_diff_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _rs_length_diff_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_card_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_scan_only_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _fully_young_cards_per_entry_ratio_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _partially_young_cards_per_entry_ratio_seq(
+ new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _partially_young_cost_per_entry_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_byte_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_byte_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _cost_per_scan_only_region_ms_during_cm_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _constant_other_time_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _young_other_cost_per_region_ms_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _non_young_other_cost_per_region_ms_seq(
+ new TruncatedSeq(TruncatedSeqLength)),
+
+ _pending_cards_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _scanned_cards_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _rs_lengths_seq(new TruncatedSeq(TruncatedSeqLength)),
+
+ _pause_time_target_ms((double) G1MaxPauseTimeMS),
+
+ //
+
+ _in_young_gc_mode(false),
+ _full_young_gcs(true),
+ _full_young_pause_num(0),
+ _partial_young_pause_num(0),
+
+ _during_marking(false),
+ _in_marking_window(false),
+ _in_marking_window_im(false),
+
+ _known_garbage_ratio(0.0),
+ _known_garbage_bytes(0),
+
+ _young_gc_eff_seq(new TruncatedSeq(TruncatedSeqLength)),
+ _target_pause_time_ms(-1.0),
+
+ _recent_prev_end_times_for_all_gcs_sec(new TruncatedSeq(NumPrevPausesForHeuristics)),
+
+ _recent_CS_bytes_used_before(new TruncatedSeq(NumPrevPausesForHeuristics)),
+ _recent_CS_bytes_surviving(new TruncatedSeq(NumPrevPausesForHeuristics)),
+
+ _recent_avg_pause_time_ratio(0.0),
+ _num_markings(0),
+ _n_marks(0),
+ _n_pauses_at_mark_end(0),
+
+ _all_full_gc_times_ms(new NumberSeq()),
+
+ _conc_refine_enabled(0),
+ _conc_refine_zero_traversals(0),
+ _conc_refine_max_traversals(0),
+ _conc_refine_current_delta(G1ConcRefineInitialDelta),
+
+ // G1PausesBtwnConcMark defaults to -1
+ // so the hack is to do the cast QQQ FIXME
+ _pauses_btwn_concurrent_mark((size_t)G1PausesBtwnConcMark),
+ _n_marks_since_last_pause(0),
+ _conc_mark_initiated(false),
+ _should_initiate_conc_mark(false),
+ _should_revert_to_full_young_gcs(false),
+ _last_full_young_gc(false),
+
+ _prev_collection_pause_used_at_end_bytes(0),
+
+ _collection_set(NULL),
+#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
+#pragma warning( disable:4355 ) // 'this' : used in base member initializer list
+#endif // _MSC_VER
+
+ _short_lived_surv_rate_group(new SurvRateGroup(this, "Short Lived",
+ G1YoungSurvRateNumRegionsSummary)),
+ _survivor_surv_rate_group(new SurvRateGroup(this, "Survivor",
+ G1YoungSurvRateNumRegionsSummary))
+ // add here any more surv rate groups
+{
+ _recent_prev_end_times_for_all_gcs_sec->add(os::elapsedTime());
+ _prev_collection_pause_end_ms = os::elapsedTime() * 1000.0;
+
+ _par_last_ext_root_scan_times_ms = new double[_parallel_gc_threads];
+ _par_last_mark_stack_scan_times_ms = new double[_parallel_gc_threads];
+ _par_last_scan_only_times_ms = new double[_parallel_gc_threads];
+ _par_last_scan_only_regions_scanned = new double[_parallel_gc_threads];
+
+ _par_last_update_rs_start_times_ms = new double[_parallel_gc_threads];
+ _par_last_update_rs_times_ms = new double[_parallel_gc_threads];
+ _par_last_update_rs_processed_buffers = new double[_parallel_gc_threads];
+
+ _par_last_scan_rs_start_times_ms = new double[_parallel_gc_threads];
+ _par_last_scan_rs_times_ms = new double[_parallel_gc_threads];
+ _par_last_scan_new_refs_times_ms = new double[_parallel_gc_threads];
+
+ _par_last_obj_copy_times_ms = new double[_parallel_gc_threads];
+
+ _par_last_termination_times_ms = new double[_parallel_gc_threads];
+
+ // we store the data from the first pass during popularity pauses
+ _pop_par_last_update_rs_start_times_ms = new double[_parallel_gc_threads];
+ _pop_par_last_update_rs_times_ms = new double[_parallel_gc_threads];
+ _pop_par_last_update_rs_processed_buffers = new double[_parallel_gc_threads];
+
+ _pop_par_last_scan_rs_start_times_ms = new double[_parallel_gc_threads];
+ _pop_par_last_scan_rs_times_ms = new double[_parallel_gc_threads];
+
+ _pop_par_last_closure_app_times_ms = new double[_parallel_gc_threads];
+
+ // start conservatively
+ _expensive_region_limit_ms = 0.5 * (double) G1MaxPauseTimeMS;
+
+ //
+
+ int index;
+ if (ParallelGCThreads == 0)
+ index = 0;
+ else if (ParallelGCThreads > 8)
+ index = 7;
+ else
+ index = ParallelGCThreads - 1;
+
+ _pending_card_diff_seq->add(0.0);
+ _rs_length_diff_seq->add(rs_length_diff_defaults[index]);
+ _cost_per_card_ms_seq->add(cost_per_card_ms_defaults[index]);
+ _cost_per_scan_only_region_ms_seq->add(
+ cost_per_scan_only_region_ms_defaults[index]);
+ _fully_young_cards_per_entry_ratio_seq->add(
+ fully_young_cards_per_entry_ratio_defaults[index]);
+ _cost_per_entry_ms_seq->add(cost_per_entry_ms_defaults[index]);
+ _cost_per_byte_ms_seq->add(cost_per_byte_ms_defaults[index]);
+ _constant_other_time_ms_seq->add(constant_other_time_ms_defaults[index]);
+ _young_other_cost_per_region_ms_seq->add(
+ young_other_cost_per_region_ms_defaults[index]);
+ _non_young_other_cost_per_region_ms_seq->add(
+ non_young_other_cost_per_region_ms_defaults[index]);
+
+ //
+
+ double time_slice = (double) G1TimeSliceMS / 1000.0;
+ double max_gc_time = (double) G1MaxPauseTimeMS / 1000.0;
+ guarantee(max_gc_time < time_slice,
+ "Max GC time should not be greater than the time slice");
+ _mmu_tracker = new G1MMUTrackerQueue(time_slice, max_gc_time);
+ _sigma = (double) G1ConfidencePerc / 100.0;
+
+ // start conservatively (around 50ms is about right)
+ _concurrent_mark_init_times_ms->add(0.05);
+ _concurrent_mark_remark_times_ms->add(0.05);
+ _concurrent_mark_cleanup_times_ms->add(0.20);
+ _tenuring_threshold = MaxTenuringThreshold;
+
+ initialize_all();
+}
+
+// Increment "i", mod "len"
+static void inc_mod(int& i, int len) {
+ i++; if (i == len) i = 0;
+}
+
+void G1CollectorPolicy::initialize_flags() {
+ set_min_alignment(HeapRegion::GrainBytes);
+ set_max_alignment(GenRemSet::max_alignment_constraint(rem_set_name()));
+ CollectorPolicy::initialize_flags();
+}
+
+void G1CollectorPolicy::init() {
+ // Set aside an initial future to_space.
+ _g1 = G1CollectedHeap::heap();
+ size_t regions = Universe::heap()->capacity() / HeapRegion::GrainBytes;
+
+ assert(Heap_lock->owned_by_self(), "Locking discipline.");
+
+ if (G1SteadyStateUsed < 50) {
+ vm_exit_during_initialization("G1SteadyStateUsed must be at least 50%.");
+ }
+ if (UseConcMarkSweepGC) {
+ vm_exit_during_initialization("-XX:+UseG1GC is incompatible with "
+ "-XX:+UseConcMarkSweepGC.");
+ }
+
+ if (G1Gen) {
+ _in_young_gc_mode = true;
+
+ if (G1YoungGenSize == 0) {
+ set_adaptive_young_list_length(true);
+ _young_list_fixed_length = 0;
+ } else {
+ set_adaptive_young_list_length(false);
+ _young_list_fixed_length = (G1YoungGenSize / HeapRegion::GrainBytes);
+ }
+ _free_regions_at_end_of_collection = _g1->free_regions();
+ _scan_only_regions_at_end_of_collection = 0;
+ calculate_young_list_min_length();
+ guarantee( _young_list_min_length == 0, "invariant, not enough info" );
+ calculate_young_list_target_config();
+ } else {
+ _young_list_fixed_length = 0;
+ _in_young_gc_mode = false;
+ }
+}
+
+void G1CollectorPolicy::calculate_young_list_min_length() {
+ _young_list_min_length = 0;
+
+ if (!adaptive_young_list_length())
+ return;
+
+ if (_alloc_rate_ms_seq->num() > 3) {
+ double now_sec = os::elapsedTime();
+ double when_ms = _mmu_tracker->when_max_gc_sec(now_sec) * 1000.0;
+ double alloc_rate_ms = predict_alloc_rate_ms();
+ int min_regions = (int) ceil(alloc_rate_ms * when_ms);
+ int current_region_num = (int) _g1->young_list_length();
+ _young_list_min_length = min_regions + current_region_num;
+ }
+}
+
+void G1CollectorPolicy::calculate_young_list_target_config() {
+ if (adaptive_young_list_length()) {
+ size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq);
+ calculate_young_list_target_config(rs_lengths);
+ } else {
+ if (full_young_gcs())
+ _young_list_target_length = _young_list_fixed_length;
+ else
+ _young_list_target_length = _young_list_fixed_length / 2;
+ _young_list_target_length = MAX2(_young_list_target_length, (size_t)1);
+ size_t so_length = calculate_optimal_so_length(_young_list_target_length);
+ guarantee( so_length < _young_list_target_length, "invariant" );
+ _young_list_so_prefix_length = so_length;
+ }
+}
+
+// This method calculate the optimal scan-only set for a fixed young
+// gen size. I couldn't work out how to reuse the more elaborate one,
+// i.e. calculate_young_list_target_config(rs_length), as the loops are
+// fundamentally different (the other one finds a config for different
+// S-O lengths, whereas here we need to do the opposite).
+size_t G1CollectorPolicy::calculate_optimal_so_length(
+ size_t young_list_length) {
+ if (!G1UseScanOnlyPrefix)
+ return 0;
+
+ if (_all_pause_times_ms->num() < 3) {
+ // we won't use a scan-only set at the beginning to allow the rest
+ // of the predictors to warm up
+ return 0;
+ }
+
+ if (_cost_per_scan_only_region_ms_seq->num() < 3) {
+ // then, we'll only set the S-O set to 1 for a little bit of time,
+ // to get enough information on the scanning cost
+ return 1;
+ }
+
+ size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq);
+ size_t rs_lengths = (size_t) get_new_prediction(_rs_lengths_seq);
+ size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff();
+ size_t scanned_cards;
+ if (full_young_gcs())
+ scanned_cards = predict_young_card_num(adj_rs_lengths);
+ else
+ scanned_cards = predict_non_young_card_num(adj_rs_lengths);
+ double base_time_ms = predict_base_elapsed_time_ms(pending_cards,
+ scanned_cards);
+
+ size_t so_length = 0;
+ double max_gc_eff = 0.0;
+ for (size_t i = 0; i < young_list_length; ++i) {
+ double gc_eff = 0.0;
+ double pause_time_ms = 0.0;
+ predict_gc_eff(young_list_length, i, base_time_ms,
+ &gc_eff, &pause_time_ms);
+ if (gc_eff > max_gc_eff) {
+ max_gc_eff = gc_eff;
+ so_length = i;
+ }
+ }
+
+ // set it to 95% of the optimal to make sure we sample the "area"
+ // around the optimal length to get up-to-date survival rate data
+ return so_length * 950 / 1000;
+}
+
+// This is a really cool piece of code! It finds the best
+// target configuration (young length / scan-only prefix length) so
+// that GC efficiency is maximized and that we also meet a pause
+// time. It's a triple nested loop. These loops are explained below
+// from the inside-out :-)
+//
+// (a) The innermost loop will try to find the optimal young length
+// for a fixed S-O length. It uses a binary search to speed up the
+// process. We assume that, for a fixed S-O length, as we add more
+// young regions to the CSet, the GC efficiency will only go up (I'll
+// skip the proof). So, using a binary search to optimize this process
+// makes perfect sense.
+//
+// (b) The middle loop will fix the S-O length before calling the
+// innermost one. It will vary it between two parameters, increasing
+// it by a given increment.
+//
+// (c) The outermost loop will call the middle loop three times.
+// (1) The first time it will explore all possible S-O length values
+// from 0 to as large as it can get, using a coarse increment (to
+// quickly "home in" to where the optimal seems to be).
+// (2) The second time it will explore the values around the optimal
+// that was found by the first iteration using a fine increment.
+// (3) Once the optimal config has been determined by the second
+// iteration, we'll redo the calculation, but setting the S-O length
+// to 95% of the optimal to make sure we sample the "area"
+// around the optimal length to get up-to-date survival rate data
+//
+// Termination conditions for the iterations are several: the pause
+// time is over the limit, we do not have enough to-space, etc.
+
+void G1CollectorPolicy::calculate_young_list_target_config(size_t rs_lengths) {
+ guarantee( adaptive_young_list_length(), "pre-condition" );
+
+ double start_time_sec = os::elapsedTime();
+ size_t min_reserve_perc = MAX2((size_t)2, (size_t)G1MinReservePerc);
+ min_reserve_perc = MIN2((size_t) 50, min_reserve_perc);
+ size_t reserve_regions =
+ (size_t) ((double) min_reserve_perc * (double) _g1->n_regions() / 100.0);
+
+ if (full_young_gcs() && _free_regions_at_end_of_collection > 0) {
+ // we are in fully-young mode and there are free regions in the heap
+
+ size_t min_so_length = 0;
+ size_t max_so_length = 0;
+
+ if (G1UseScanOnlyPrefix) {
+ if (_all_pause_times_ms->num() < 3) {
+ // we won't use a scan-only set at the beginning to allow the rest
+ // of the predictors to warm up
+ min_so_length = 0;
+ max_so_length = 0;
+ } else if (_cost_per_scan_only_region_ms_seq->num() < 3) {
+ // then, we'll only set the S-O set to 1 for a little bit of time,
+ // to get enough information on the scanning cost
+ min_so_length = 1;
+ max_so_length = 1;
+ } else if (_in_marking_window || _last_full_young_gc) {
+ // no S-O prefix during a marking phase either, as at the end
+ // of the marking phase we'll have to use a very small young
+ // length target to fill up the rest of the CSet with
+ // non-young regions and, if we have lots of scan-only regions
+ // left-over, we will not be able to add any more non-young
+ // regions.
+ min_so_length = 0;
+ max_so_length = 0;
+ } else {
+ // this is the common case; we'll never reach the maximum, we
+ // one of the end conditions will fire well before that
+ // (hopefully!)
+ min_so_length = 0;
+ max_so_length = _free_regions_at_end_of_collection - 1;
+ }
+ } else {
+ // no S-O prefix, as the switch is not set, but we still need to
+ // do one iteration to calculate the best young target that
+ // meets the pause time; this way we reuse the same code instead
+ // of replicating it
+ min_so_length = 0;
+ max_so_length = 0;
+ }
+
+ double target_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0;
+ size_t pending_cards = (size_t) get_new_prediction(_pending_cards_seq);
+ size_t adj_rs_lengths = rs_lengths + predict_rs_length_diff();
+ size_t scanned_cards;
+ if (full_young_gcs())
+ scanned_cards = predict_young_card_num(adj_rs_lengths);
+ else
+ scanned_cards = predict_non_young_card_num(adj_rs_lengths);
+ // calculate this once, so that we don't have to recalculate it in
+ // the innermost loop
+ double base_time_ms = predict_base_elapsed_time_ms(pending_cards,
+ scanned_cards);
+
+ // the result
+ size_t final_young_length = 0;
+ size_t final_so_length = 0;
+ double final_gc_eff = 0.0;
+ // we'll also keep track of how many times we go into the inner loop
+ // this is for profiling reasons
+ size_t calculations = 0;
+
+ // this determines which of the three iterations the outer loop is in
+ typedef enum {
+ pass_type_coarse,
+ pass_type_fine,
+ pass_type_final
+ } pass_type_t;
+
+ // range of the outer loop's iteration
+ size_t from_so_length = min_so_length;
+ size_t to_so_length = max_so_length;
+ guarantee( from_so_length <= to_so_length, "invariant" );
+
+ // this will keep the S-O length that's found by the second
+ // iteration of the outer loop; we'll keep it just in case the third
+ // iteration fails to find something
+ size_t fine_so_length = 0;
+
+ // the increment step for the coarse (first) iteration
+ size_t so_coarse_increments = 5;
+
+ // the common case, we'll start with the coarse iteration
+ pass_type_t pass = pass_type_coarse;
+ size_t so_length_incr = so_coarse_increments;
+
+ if (from_so_length == to_so_length) {
+ // not point in doing the coarse iteration, we'll go directly into
+ // the fine one (we essentially trying to find the optimal young
+ // length for a fixed S-O length).
+ so_length_incr = 1;
+ pass = pass_type_final;
+ } else if (to_so_length - from_so_length < 3 * so_coarse_increments) {
+ // again, the range is too short so no point in foind the coarse
+ // iteration either
+ so_length_incr = 1;
+ pass = pass_type_fine;
+ }
+
+ bool done = false;
+ // this is the outermost loop
+ while (!done) {
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr("searching between " SIZE_FORMAT " and " SIZE_FORMAT
+ ", incr " SIZE_FORMAT ", pass %s",
+ from_so_length, to_so_length, so_length_incr,
+ (pass == pass_type_coarse) ? "coarse" :
+ (pass == pass_type_fine) ? "fine" : "final");
+#endif // 0
+
+ size_t so_length = from_so_length;
+ size_t init_free_regions =
+ MAX2((size_t)0,
+ _free_regions_at_end_of_collection +
+ _scan_only_regions_at_end_of_collection - reserve_regions);
+
+ // this determines whether a configuration was found
+ bool gc_eff_set = false;
+ // this is the middle loop
+ while (so_length <= to_so_length) {
+ // base time, which excludes region-related time; again we
+ // calculate it once to avoid recalculating it in the
+ // innermost loop
+ double base_time_with_so_ms =
+ base_time_ms + predict_scan_only_time_ms(so_length);
+ // it's already over the pause target, go around
+ if (base_time_with_so_ms > target_pause_time_ms)
+ break;
+
+ size_t starting_young_length = so_length+1;
+
+ // we make sure that the short young length that makes sense
+ // (one more than the S-O length) is feasible
+ size_t min_young_length = starting_young_length;
+ double min_gc_eff;
+ bool min_ok;
+ ++calculations;
+ min_ok = predict_gc_eff(min_young_length, so_length,
+ base_time_with_so_ms,
+ init_free_regions, target_pause_time_ms,
+ &min_gc_eff);
+
+ if (min_ok) {
+ // the shortest young length is indeed feasible; we'll know
+ // set up the max young length and we'll do a binary search
+ // between min_young_length and max_young_length
+ size_t max_young_length = _free_regions_at_end_of_collection - 1;
+ double max_gc_eff = 0.0;
+ bool max_ok = false;
+
+ // the innermost loop! (finally!)
+ while (max_young_length > min_young_length) {
+ // we'll make sure that min_young_length is always at a
+ // feasible config
+ guarantee( min_ok, "invariant" );
+
+ ++calculations;
+ max_ok = predict_gc_eff(max_young_length, so_length,
+ base_time_with_so_ms,
+ init_free_regions, target_pause_time_ms,
+ &max_gc_eff);
+
+ size_t diff = (max_young_length - min_young_length) / 2;
+ if (max_ok) {
+ min_young_length = max_young_length;
+ min_gc_eff = max_gc_eff;
+ min_ok = true;
+ }
+ max_young_length = min_young_length + diff;
+ }
+
+ // the innermost loop found a config
+ guarantee( min_ok, "invariant" );
+ if (min_gc_eff > final_gc_eff) {
+ // it's the best config so far, so we'll keep it
+ final_gc_eff = min_gc_eff;
+ final_young_length = min_young_length;
+ final_so_length = so_length;
+ gc_eff_set = true;
+ }
+ }
+
+ // incremental the fixed S-O length and go around
+ so_length += so_length_incr;
+ }
+
+ // this is the end of the outermost loop and we need to decide
+ // what to do during the next iteration
+ if (pass == pass_type_coarse) {
+ // we just did the coarse pass (first iteration)
+
+ if (!gc_eff_set)
+ // we didn't find a feasible config so we'll just bail out; of
+ // course, it might be the case that we missed it; but I'd say
+ // it's a bit unlikely
+ done = true;
+ else {
+ // We did find a feasible config with optimal GC eff during
+ // the first pass. So the second pass we'll only consider the
+ // S-O lengths around that config with a fine increment.
+
+ guarantee( so_length_incr == so_coarse_increments, "invariant" );
+ guarantee( final_so_length >= min_so_length, "invariant" );
+
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr(" coarse pass: SO length " SIZE_FORMAT,
+ final_so_length);
+#endif // 0
+
+ from_so_length =
+ (final_so_length - min_so_length > so_coarse_increments) ?
+ final_so_length - so_coarse_increments + 1 : min_so_length;
+ to_so_length =
+ (max_so_length - final_so_length > so_coarse_increments) ?
+ final_so_length + so_coarse_increments - 1 : max_so_length;
+
+ pass = pass_type_fine;
+ so_length_incr = 1;
+ }
+ } else if (pass == pass_type_fine) {
+ // we just finished the second pass
+
+ if (!gc_eff_set) {
+ // we didn't find a feasible config (yes, it's possible;
+ // notice that, sometimes, we go directly into the fine
+ // iteration and skip the coarse one) so we bail out
+ done = true;
+ } else {
+ // We did find a feasible config with optimal GC eff
+ guarantee( so_length_incr == 1, "invariant" );
+
+ if (final_so_length == 0) {
+ // The config is of an empty S-O set, so we'll just bail out
+ done = true;
+ } else {
+ // we'll go around once more, setting the S-O length to 95%
+ // of the optimal
+ size_t new_so_length = 950 * final_so_length / 1000;
+
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr(" fine pass: SO length " SIZE_FORMAT
+ ", setting it to " SIZE_FORMAT,
+ final_so_length, new_so_length);
+#endif // 0
+
+ from_so_length = new_so_length;
+ to_so_length = new_so_length;
+ fine_so_length = final_so_length;
+
+ pass = pass_type_final;
+ }
+ }
+ } else if (pass == pass_type_final) {
+ // we just finished the final (third) pass
+
+ if (!gc_eff_set)
+ // we didn't find a feasible config, so we'll just use the one
+ // we found during the second pass, which we saved
+ final_so_length = fine_so_length;
+
+ // and we're done!
+ done = true;
+ } else {
+ guarantee( false, "should never reach here" );
+ }
+
+ // we now go around the outermost loop
+ }
+
+ // we should have at least one region in the target young length
+ _young_list_target_length = MAX2((size_t) 1, final_young_length);
+ if (final_so_length >= final_young_length)
+ // and we need to ensure that the S-O length is not greater than
+ // the target young length (this is being a bit careful)
+ final_so_length = 0;
+ _young_list_so_prefix_length = final_so_length;
+ guarantee( !_in_marking_window || !_last_full_young_gc ||
+ _young_list_so_prefix_length == 0, "invariant" );
+
+ // let's keep an eye of how long we spend on this calculation
+ // right now, I assume that we'll print it when we need it; we
+ // should really adde it to the breakdown of a pause
+ double end_time_sec = os::elapsedTime();
+ double elapsed_time_ms = (end_time_sec - start_time_sec) * 1000.0;
+
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr("target = %1.1lf ms, young = " SIZE_FORMAT
+ ", SO = " SIZE_FORMAT ", "
+ "elapsed %1.2lf ms, calcs: " SIZE_FORMAT " (%s%s) "
+ SIZE_FORMAT SIZE_FORMAT,
+ target_pause_time_ms,
+ _young_list_target_length - _young_list_so_prefix_length,
+ _young_list_so_prefix_length,
+ elapsed_time_ms,
+ calculations,
+ full_young_gcs() ? "full" : "partial",
+ should_initiate_conc_mark() ? " i-m" : "",
+ in_marking_window(),
+ in_marking_window_im());
+#endif // 0
+
+ if (_young_list_target_length < _young_list_min_length) {
+ // bummer; this means that, if we do a pause when the optimal
+ // config dictates, we'll violate the pause spacing target (the
+ // min length was calculate based on the application's current
+ // alloc rate);
+
+ // so, we have to bite the bullet, and allocate the minimum
+ // number. We'll violate our target, but we just can't meet it.
+
+ size_t so_length = 0;
+ // a note further up explains why we do not want an S-O length
+ // during marking
+ if (!_in_marking_window && !_last_full_young_gc)
+ // but we can still try to see whether we can find an optimal
+ // S-O length
+ so_length = calculate_optimal_so_length(_young_list_min_length);
+
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr("adjusted target length from "
+ SIZE_FORMAT " to " SIZE_FORMAT
+ ", SO " SIZE_FORMAT,
+ _young_list_target_length, _young_list_min_length,
+ so_length);
+#endif // 0
+
+ _young_list_target_length =
+ MAX2(_young_list_min_length, (size_t)1);
+ _young_list_so_prefix_length = so_length;
+ }
+ } else {
+ // we are in a partially-young mode or we've run out of regions (due
+ // to evacuation failure)
+
+#if 0
+ // leave this in for debugging, just in case
+ gclog_or_tty->print_cr("(partial) setting target to " SIZE_FORMAT
+ ", SO " SIZE_FORMAT,
+ _young_list_min_length, 0);
+#endif // 0
+
+ // we'll do the pause as soon as possible and with no S-O prefix
+ // (see above for the reasons behind the latter)
+ _young_list_target_length =
+ MAX2(_young_list_min_length, (size_t) 1);
+ _young_list_so_prefix_length = 0;
+ }
+
+ _rs_lengths_prediction = rs_lengths;
+}
+
+// This is used by: calculate_optimal_so_length(length). It returns
+// the GC eff and predicted pause time for a particular config
+void
+G1CollectorPolicy::predict_gc_eff(size_t young_length,
+ size_t so_length,
+ double base_time_ms,
+ double* ret_gc_eff,
+ double* ret_pause_time_ms) {
+ double so_time_ms = predict_scan_only_time_ms(so_length);
+ double accum_surv_rate_adj = 0.0;
+ if (so_length > 0)
+ accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1));
+ double accum_surv_rate =
+ accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj;
+ size_t bytes_to_copy =
+ (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes);
+ double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy);
+ double young_other_time_ms =
+ predict_young_other_time_ms(young_length - so_length);
+ double pause_time_ms =
+ base_time_ms + so_time_ms + copy_time_ms + young_other_time_ms;
+ size_t reclaimed_bytes =
+ (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy;
+ double gc_eff = (double) reclaimed_bytes / pause_time_ms;
+
+ *ret_gc_eff = gc_eff;
+ *ret_pause_time_ms = pause_time_ms;
+}
+
+// This is used by: calculate_young_list_target_config(rs_length). It
+// returns the GC eff of a particular config. It returns false if that
+// config violates any of the end conditions of the search in the
+// calling method, or true upon success. The end conditions were put
+// here since it's called twice and it was best not to replicate them
+// in the caller. Also, passing the parameteres avoids having to
+// recalculate them in the innermost loop.
+bool
+G1CollectorPolicy::predict_gc_eff(size_t young_length,
+ size_t so_length,
+ double base_time_with_so_ms,
+ size_t init_free_regions,
+ double target_pause_time_ms,
+ double* ret_gc_eff) {
+ *ret_gc_eff = 0.0;
+
+ if (young_length >= init_free_regions)
+ // end condition 1: not enough space for the young regions
+ return false;
+
+ double accum_surv_rate_adj = 0.0;
+ if (so_length > 0)
+ accum_surv_rate_adj = accum_yg_surv_rate_pred((int)(so_length - 1));
+ double accum_surv_rate =
+ accum_yg_surv_rate_pred((int)(young_length - 1)) - accum_surv_rate_adj;
+ size_t bytes_to_copy =
+ (size_t) (accum_surv_rate * (double) HeapRegion::GrainBytes);
+ double copy_time_ms = predict_object_copy_time_ms(bytes_to_copy);
+ double young_other_time_ms =
+ predict_young_other_time_ms(young_length - so_length);
+ double pause_time_ms =
+ base_time_with_so_ms + copy_time_ms + young_other_time_ms;
+
+ if (pause_time_ms > target_pause_time_ms)
+ // end condition 2: over the target pause time
+ return false;
+
+ size_t reclaimed_bytes =
+ (young_length - so_length) * HeapRegion::GrainBytes - bytes_to_copy;
+ size_t free_bytes =
+ (init_free_regions - young_length) * HeapRegion::GrainBytes;
+
+ if ((2.0 + sigma()) * (double) bytes_to_copy > (double) free_bytes)
+ // end condition 3: out of to-space (conservatively)
+ return false;
+
+ // success!
+ double gc_eff = (double) reclaimed_bytes / pause_time_ms;
+ *ret_gc_eff = gc_eff;
+
+ return true;
+}
+
+void G1CollectorPolicy::check_prediction_validity() {
+ guarantee( adaptive_young_list_length(), "should not call this otherwise" );
+
+ size_t rs_lengths = _g1->young_list_sampled_rs_lengths();
+ if (rs_lengths > _rs_lengths_prediction) {
+ // add 10% to avoid having to recalculate often
+ size_t rs_lengths_prediction = rs_lengths * 1100 / 1000;
+ calculate_young_list_target_config(rs_lengths_prediction);
+ }
+}
+
+HeapWord* G1CollectorPolicy::mem_allocate_work(size_t size,
+ bool is_tlab,
+ bool* gc_overhead_limit_was_exceeded) {
+ guarantee(false, "Not using this policy feature yet.");
+ return NULL;
+}
+
+// This method controls how a collector handles one or more
+// of its generations being fully allocated.
+HeapWord* G1CollectorPolicy::satisfy_failed_allocation(size_t size,
+ bool is_tlab) {
+ guarantee(false, "Not using this policy feature yet.");
+ return NULL;
+}
+
+
+#ifndef PRODUCT
+bool G1CollectorPolicy::verify_young_ages() {
+ HeapRegion* head = _g1->young_list_first_region();
+ return
+ verify_young_ages(head, _short_lived_surv_rate_group);
+ // also call verify_young_ages on any additional surv rate groups
+}
+
+bool
+G1CollectorPolicy::verify_young_ages(HeapRegion* head,
+ SurvRateGroup *surv_rate_group) {
+ guarantee( surv_rate_group != NULL, "pre-condition" );
+
+ const char* name = surv_rate_group->name();
+ bool ret = true;
+ int prev_age = -1;
+
+ for (HeapRegion* curr = head;
+ curr != NULL;
+ curr = curr->get_next_young_region()) {
+ SurvRateGroup* group = curr->surv_rate_group();
+ if (group == NULL && !curr->is_survivor()) {
+ gclog_or_tty->print_cr("## %s: encountered NULL surv_rate_group", name);
+ ret = false;
+ }
+
+ if (surv_rate_group == group) {
+ int age = curr->age_in_surv_rate_group();
+
+ if (age < 0) {
+ gclog_or_tty->print_cr("## %s: encountered negative age", name);
+ ret = false;
+ }
+
+ if (age <= prev_age) {
+ gclog_or_tty->print_cr("## %s: region ages are not strictly increasing "
+ "(%d, %d)", name, age, prev_age);
+ ret = false;
+ }
+ prev_age = age;
+ }
+ }
+
+ return ret;
+}
+#endif // PRODUCT
+
+void G1CollectorPolicy::record_full_collection_start() {
+ _cur_collection_start_sec = os::elapsedTime();
+ // Release the future to-space so that it is available for compaction into.
+ _g1->set_full_collection();
+}
+
+void G1CollectorPolicy::record_full_collection_end() {
+ // Consider this like a collection pause for the purposes of allocation
+ // since last pause.
+ double end_sec = os::elapsedTime();
+ double full_gc_time_sec = end_sec - _cur_collection_start_sec;
+ double full_gc_time_ms = full_gc_time_sec * 1000.0;
+
+ checkpoint_conc_overhead();
+
+ _all_full_gc_times_ms->add(full_gc_time_ms);
+
+ update_recent_gc_times(end_sec, full_gc_time_sec);
+
+ _g1->clear_full_collection();
+
+ // "Nuke" the heuristics that control the fully/partially young GC
+ // transitions and make sure we start with fully young GCs after the
+ // Full GC.
+ set_full_young_gcs(true);
+ _last_full_young_gc = false;
+ _should_revert_to_full_young_gcs = false;
+ _should_initiate_conc_mark = false;
+ _known_garbage_bytes = 0;
+ _known_garbage_ratio = 0.0;
+ _in_marking_window = false;
+ _in_marking_window_im = false;
+
+ _short_lived_surv_rate_group->record_scan_only_prefix(0);
+ _short_lived_surv_rate_group->start_adding_regions();
+ // also call this on any additional surv rate groups
+
+ _prev_region_num_young = _region_num_young;
+ _prev_region_num_tenured = _region_num_tenured;
+
+ _free_regions_at_end_of_collection = _g1->free_regions();
+ _scan_only_regions_at_end_of_collection = 0;
+ calculate_young_list_min_length();
+ calculate_young_list_target_config();
+ }
+
+void G1CollectorPolicy::record_pop_compute_rc_start() {
+ _pop_compute_rc_start = os::elapsedTime();
+}
+void G1CollectorPolicy::record_pop_compute_rc_end() {
+ double ms = (os::elapsedTime() - _pop_compute_rc_start)*1000.0;
+ _cur_popular_compute_rc_time_ms = ms;
+ _pop_compute_rc_start = 0.0;
+}
+void G1CollectorPolicy::record_pop_evac_start() {
+ _pop_evac_start = os::elapsedTime();
+}
+void G1CollectorPolicy::record_pop_evac_end() {
+ double ms = (os::elapsedTime() - _pop_evac_start)*1000.0;
+ _cur_popular_evac_time_ms = ms;
+ _pop_evac_start = 0.0;
+}
+
+void G1CollectorPolicy::record_before_bytes(size_t bytes) {
+ _bytes_in_to_space_before_gc += bytes;
+}
+
+void G1CollectorPolicy::record_after_bytes(size_t bytes) {
+ _bytes_in_to_space_after_gc += bytes;
+}
+
+void G1CollectorPolicy::record_stop_world_start() {
+ _stop_world_start = os::elapsedTime();
+}
+
+void G1CollectorPolicy::record_collection_pause_start(double start_time_sec,
+ size_t start_used) {
+ if (PrintGCDetails) {
+ gclog_or_tty->stamp(PrintGCTimeStamps);
+ gclog_or_tty->print("[GC pause");
+ if (in_young_gc_mode())
+ gclog_or_tty->print(" (%s)", full_young_gcs() ? "young" : "partial");
+ }
+
+ assert(_g1->used_regions() == _g1->recalculate_used_regions(),
+ "sanity");
+
+ double s_w_t_ms = (start_time_sec - _stop_world_start) * 1000.0;
+ _all_stop_world_times_ms->add(s_w_t_ms);
+ _stop_world_start = 0.0;
+
+ _cur_collection_start_sec = start_time_sec;
+ _cur_collection_pause_used_at_start_bytes = start_used;
+ _cur_collection_pause_used_regions_at_start = _g1->used_regions();
+ _pending_cards = _g1->pending_card_num();
+ _max_pending_cards = _g1->max_pending_card_num();
+
+ _bytes_in_to_space_before_gc = 0;
+ _bytes_in_to_space_after_gc = 0;
+ _bytes_in_collection_set_before_gc = 0;
+
+#ifdef DEBUG
+ // initialise these to something well known so that we can spot
+ // if they are not set properly
+
+ for (int i = 0; i < _parallel_gc_threads; ++i) {
+ _par_last_ext_root_scan_times_ms[i] = -666.0;
+ _par_last_mark_stack_scan_times_ms[i] = -666.0;
+ _par_last_scan_only_times_ms[i] = -666.0;
+ _par_last_scan_only_regions_scanned[i] = -666.0;
+ _par_last_update_rs_start_times_ms[i] = -666.0;
+ _par_last_update_rs_times_ms[i] = -666.0;
+ _par_last_update_rs_processed_buffers[i] = -666.0;
+ _par_last_scan_rs_start_times_ms[i] = -666.0;
+ _par_last_scan_rs_times_ms[i] = -666.0;
+ _par_last_scan_new_refs_times_ms[i] = -666.0;
+ _par_last_obj_copy_times_ms[i] = -666.0;
+ _par_last_termination_times_ms[i] = -666.0;
+
+ _pop_par_last_update_rs_start_times_ms[i] = -666.0;
+ _pop_par_last_update_rs_times_ms[i] = -666.0;
+ _pop_par_last_update_rs_processed_buffers[i] = -666.0;
+ _pop_par_last_scan_rs_start_times_ms[i] = -666.0;
+ _pop_par_last_scan_rs_times_ms[i] = -666.0;
+ _pop_par_last_closure_app_times_ms[i] = -666.0;
+ }
+#endif
+
+ for (int i = 0; i < _aux_num; ++i) {
+ _cur_aux_times_ms[i] = 0.0;
+ _cur_aux_times_set[i] = false;
+ }
+
+ _satb_drain_time_set = false;
+ _last_satb_drain_processed_buffers = -1;
+
+ if (in_young_gc_mode())
+ _last_young_gc_full = false;
+
+
+ // do that for any other surv rate groups
+ _short_lived_surv_rate_group->stop_adding_regions();
+ size_t short_lived_so_length = _young_list_so_prefix_length;
+ _short_lived_surv_rate_group->record_scan_only_prefix(short_lived_so_length);
+ tag_scan_only(short_lived_so_length);
+
+ assert( verify_young_ages(), "region age verification" );
+}
+
+void G1CollectorPolicy::tag_scan_only(size_t short_lived_scan_only_length) {
+ // done in a way that it can be extended for other surv rate groups too...
+
+ HeapRegion* head = _g1->young_list_first_region();
+ bool finished_short_lived = (short_lived_scan_only_length == 0);
+
+ if (finished_short_lived)
+ return;
+
+ for (HeapRegion* curr = head;
+ curr != NULL;
+ curr = curr->get_next_young_region()) {
+ SurvRateGroup* surv_rate_group = curr->surv_rate_group();
+ int age = curr->age_in_surv_rate_group();
+
+ if (surv_rate_group == _short_lived_surv_rate_group) {
+ if ((size_t)age < short_lived_scan_only_length)
+ curr->set_scan_only();
+ else
+ finished_short_lived = true;
+ }
+
+
+ if (finished_short_lived)
+ return;
+ }
+
+ guarantee( false, "we should never reach here" );
+}
+
+void G1CollectorPolicy::record_popular_pause_preamble_start() {
+ _cur_popular_preamble_start_ms = os::elapsedTime() * 1000.0;
+}
+
+void G1CollectorPolicy::record_popular_pause_preamble_end() {
+ _cur_popular_preamble_time_ms =
+ (os::elapsedTime() * 1000.0) - _cur_popular_preamble_start_ms;
+
+ // copy the recorded statistics of the first pass to temporary arrays
+ for (int i = 0; i < _parallel_gc_threads; ++i) {
+ _pop_par_last_update_rs_start_times_ms[i] = _par_last_update_rs_start_times_ms[i];
+ _pop_par_last_update_rs_times_ms[i] = _par_last_update_rs_times_ms[i];
+ _pop_par_last_update_rs_processed_buffers[i] = _par_last_update_rs_processed_buffers[i];
+ _pop_par_last_scan_rs_start_times_ms[i] = _par_last_scan_rs_start_times_ms[i];
+ _pop_par_last_scan_rs_times_ms[i] = _par_last_scan_rs_times_ms[i];
+ _pop_par_last_closure_app_times_ms[i] = _par_last_obj_copy_times_ms[i];
+ }
+}
+
+void G1CollectorPolicy::record_mark_closure_time(double mark_closure_time_ms) {
+ _mark_closure_time_ms = mark_closure_time_ms;
+}
+
+void G1CollectorPolicy::record_concurrent_mark_init_start() {
+ _mark_init_start_sec = os::elapsedTime();
+ guarantee(!in_young_gc_mode(), "should not do be here in young GC mode");
+}
+
+void G1CollectorPolicy::record_concurrent_mark_init_end_pre(double
+ mark_init_elapsed_time_ms) {
+ _during_marking = true;
+ _should_initiate_conc_mark = false;
+ _cur_mark_stop_world_time_ms = mark_init_elapsed_time_ms;
+}
+
+void G1CollectorPolicy::record_concurrent_mark_init_end() {
+ double end_time_sec = os::elapsedTime();
+ double elapsed_time_ms = (end_time_sec - _mark_init_start_sec) * 1000.0;
+ _concurrent_mark_init_times_ms->add(elapsed_time_ms);
+ checkpoint_conc_overhead();
+ record_concurrent_mark_init_end_pre(elapsed_time_ms);
+
+ _mmu_tracker->add_pause(_mark_init_start_sec, end_time_sec, true);
+}
+
+void G1CollectorPolicy::record_concurrent_mark_remark_start() {
+ _mark_remark_start_sec = os::elapsedTime();
+ _during_marking = false;
+}
+
+void G1CollectorPolicy::record_concurrent_mark_remark_end() {
+ double end_time_sec = os::elapsedTime();
+ double elapsed_time_ms = (end_time_sec - _mark_remark_start_sec)*1000.0;
+ checkpoint_conc_overhead();
+ _concurrent_mark_remark_times_ms->add(elapsed_time_ms);
+ _cur_mark_stop_world_time_ms += elapsed_time_ms;
+ _prev_collection_pause_end_ms += elapsed_time_ms;
+
+ _mmu_tracker->add_pause(_mark_remark_start_sec, end_time_sec, true);
+}
+
+void G1CollectorPolicy::record_concurrent_mark_cleanup_start() {
+ _mark_cleanup_start_sec = os::elapsedTime();
+}
+
+void
+G1CollectorPolicy::record_concurrent_mark_cleanup_end(size_t freed_bytes,
+ size_t max_live_bytes) {
+ record_concurrent_mark_cleanup_end_work1(freed_bytes, max_live_bytes);
+ record_concurrent_mark_cleanup_end_work2();
+}
+
+void
+G1CollectorPolicy::
+record_concurrent_mark_cleanup_end_work1(size_t freed_bytes,
+ size_t max_live_bytes) {
+ if (_n_marks < 2) _n_marks++;
+ if (G1PolicyVerbose > 0)
+ gclog_or_tty->print_cr("At end of marking, max_live is " SIZE_FORMAT " MB "
+ " (of " SIZE_FORMAT " MB heap).",
+ max_live_bytes/M, _g1->capacity()/M);
+}
+
+// The important thing about this is that it includes "os::elapsedTime".
+void G1CollectorPolicy::record_concurrent_mark_cleanup_end_work2() {
+ checkpoint_conc_overhead();
+ double end_time_sec = os::elapsedTime();
+ double elapsed_time_ms = (end_time_sec - _mark_cleanup_start_sec)*1000.0;
+ _concurrent_mark_cleanup_times_ms->add(elapsed_time_ms);
+ _cur_mark_stop_world_time_ms += elapsed_time_ms;
+ _prev_collection_pause_end_ms += elapsed_time_ms;
+
+ _mmu_tracker->add_pause(_mark_cleanup_start_sec, end_time_sec, true);
+
+ _num_markings++;
+
+ // We did a marking, so reset the "since_last_mark" variables.
+ double considerConcMarkCost = 1.0;
+ // If there are available processors, concurrent activity is free...
+ if (Threads::number_of_non_daemon_threads() * 2 <
+ os::active_processor_count()) {
+ considerConcMarkCost = 0.0;
+ }
+ _n_pauses_at_mark_end = _n_pauses;
+ _n_marks_since_last_pause++;
+ _conc_mark_initiated = false;
+}
+
+void
+G1CollectorPolicy::record_concurrent_mark_cleanup_completed() {
+ if (in_young_gc_mode()) {
+ _should_revert_to_full_young_gcs = false;
+ _last_full_young_gc = true;
+ _in_marking_window = false;
+ if (adaptive_young_list_length())
+ calculate_young_list_target_config();
+ }
+}
+
+void G1CollectorPolicy::record_concurrent_pause() {
+ if (_stop_world_start > 0.0) {
+ double yield_ms = (os::elapsedTime() - _stop_world_start) * 1000.0;
+ _all_yield_times_ms->add(yield_ms);
+ }
+}
+
+void G1CollectorPolicy::record_concurrent_pause_end() {
+}
+
+void G1CollectorPolicy::record_collection_pause_end_CH_strong_roots() {
+ _cur_CH_strong_roots_end_sec = os::elapsedTime();
+ _cur_CH_strong_roots_dur_ms =
+ (_cur_CH_strong_roots_end_sec - _cur_collection_start_sec) * 1000.0;
+}
+
+void G1CollectorPolicy::record_collection_pause_end_G1_strong_roots() {
+ _cur_G1_strong_roots_end_sec = os::elapsedTime();
+ _cur_G1_strong_roots_dur_ms =
+ (_cur_G1_strong_roots_end_sec - _cur_CH_strong_roots_end_sec) * 1000.0;
+}
+
+template
+T sum_of(T* sum_arr, int start, int n, int N) {
+ T sum = (T)0;
+ for (int i = 0; i < n; i++) {
+ int j = (start + i) % N;
+ sum += sum_arr[j];
+ }
+ return sum;
+}
+
+void G1CollectorPolicy::print_par_stats (int level,
+ const char* str,
+ double* data,
+ bool summary) {
+ double min = data[0], max = data[0];
+ double total = 0.0;
+ int j;
+ for (j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print("[%s (ms):", str);
+ for (uint i = 0; i < ParallelGCThreads; ++i) {
+ double val = data[i];
+ if (val < min)
+ min = val;
+ if (val > max)
+ max = val;
+ total += val;
+ gclog_or_tty->print(" %3.1lf", val);
+ }
+ if (summary) {
+ gclog_or_tty->print_cr("");
+ double avg = total / (double) ParallelGCThreads;
+ gclog_or_tty->print(" ");
+ for (j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print("Avg: %5.1lf, Min: %5.1lf, Max: %5.1lf",
+ avg, min, max);
+ }
+ gclog_or_tty->print_cr("]");
+}
+
+void G1CollectorPolicy::print_par_buffers (int level,
+ const char* str,
+ double* data,
+ bool summary) {
+ double min = data[0], max = data[0];
+ double total = 0.0;
+ int j;
+ for (j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print("[%s :", str);
+ for (uint i = 0; i < ParallelGCThreads; ++i) {
+ double val = data[i];
+ if (val < min)
+ min = val;
+ if (val > max)
+ max = val;
+ total += val;
+ gclog_or_tty->print(" %d", (int) val);
+ }
+ if (summary) {
+ gclog_or_tty->print_cr("");
+ double avg = total / (double) ParallelGCThreads;
+ gclog_or_tty->print(" ");
+ for (j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print("Sum: %d, Avg: %d, Min: %d, Max: %d",
+ (int)total, (int)avg, (int)min, (int)max);
+ }
+ gclog_or_tty->print_cr("]");
+}
+
+void G1CollectorPolicy::print_stats (int level,
+ const char* str,
+ double value) {
+ for (int j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print_cr("[%s: %5.1lf ms]", str, value);
+}
+
+void G1CollectorPolicy::print_stats (int level,
+ const char* str,
+ int value) {
+ for (int j = 0; j < level; ++j)
+ gclog_or_tty->print(" ");
+ gclog_or_tty->print_cr("[%s: %d]", str, value);
+}
+
+double G1CollectorPolicy::avg_value (double* data) {
+ if (ParallelGCThreads > 0) {
+ double ret = 0.0;
+ for (uint i = 0; i < ParallelGCThreads; ++i)
+ ret += data[i];
+ return ret / (double) ParallelGCThreads;
+ } else {
+ return data[0];
+ }
+}
+
+double G1CollectorPolicy::max_value (double* data) {
+ if (ParallelGCThreads > 0) {
+ double ret = data[0];
+ for (uint i = 1; i < ParallelGCThreads; ++i)
+ if (data[i] > ret)
+ ret = data[i];
+ return ret;
+ } else {
+ return data[0];
+ }
+}
+
+double G1CollectorPolicy::sum_of_values (double* data) {
+ if (ParallelGCThreads > 0) {
+ double sum = 0.0;
+ for (uint i = 0; i < ParallelGCThreads; i++)
+ sum += data[i];
+ return sum;
+ } else {
+ return data[0];
+ }
+}
+
+double G1CollectorPolicy::max_sum (double* data1,
+ double* data2) {
+ double ret = data1[0] + data2[0];
+
+ if (ParallelGCThreads > 0) {
+ for (uint i = 1; i < ParallelGCThreads; ++i) {
+ double data = data1[i] + data2[i];
+ if (data > ret)
+ ret = data;
+ }
+ }
+ return ret;
+}
+
+// Anything below that is considered to be zero
+#define MIN_TIMER_GRANULARITY 0.0000001
+
+void G1CollectorPolicy::record_collection_pause_end(bool popular,
+ bool abandoned) {
+ double end_time_sec = os::elapsedTime();
+ double elapsed_ms = _last_pause_time_ms;
+ bool parallel = ParallelGCThreads > 0;
+ double evac_ms = (end_time_sec - _cur_G1_strong_roots_end_sec) * 1000.0;
+ size_t rs_size =
+ _cur_collection_pause_used_regions_at_start - collection_set_size();
+ size_t cur_used_bytes = _g1->used();
+ assert(cur_used_bytes == _g1->recalculate_used(), "It should!");
+ bool last_pause_included_initial_mark = false;
+
+#ifndef PRODUCT
+ if (G1YoungSurvRateVerbose) {
+ gclog_or_tty->print_cr("");
+ _short_lived_surv_rate_group->print();
+ // do that for any other surv rate groups too
+ }
+#endif // PRODUCT
+
+ checkpoint_conc_overhead();
+
+ if (in_young_gc_mode()) {
+ last_pause_included_initial_mark = _should_initiate_conc_mark;
+ if (last_pause_included_initial_mark)
+ record_concurrent_mark_init_end_pre(0.0);
+
+ size_t min_used_targ =
+ (_g1->capacity() / 100) * (G1SteadyStateUsed - G1SteadyStateUsedDelta);
+
+ if (cur_used_bytes > min_used_targ) {
+ if (cur_used_bytes <= _prev_collection_pause_used_at_end_bytes) {
+ } else if (!_g1->mark_in_progress() && !_last_full_young_gc) {
+ _should_initiate_conc_mark = true;
+ }
+ }
+
+ _prev_collection_pause_used_at_end_bytes = cur_used_bytes;
+ }
+
+ _mmu_tracker->add_pause(end_time_sec - elapsed_ms/1000.0,
+ end_time_sec, false);
+
+ guarantee(_cur_collection_pause_used_regions_at_start >=
+ collection_set_size(),
+ "Negative RS size?");
+
+ // This assert is exempted when we're doing parallel collection pauses,
+ // because the fragmentation caused by the parallel GC allocation buffers
+ // can lead to more memory being used during collection than was used
+ // before. Best leave this out until the fragmentation problem is fixed.
+ // Pauses in which evacuation failed can also lead to negative
+ // collections, since no space is reclaimed from a region containing an
+ // object whose evacuation failed.
+ // Further, we're now always doing parallel collection. But I'm still
+ // leaving this here as a placeholder for a more precise assertion later.
+ // (DLD, 10/05.)
+ assert((true || parallel) // Always using GC LABs now.
+ || _g1->evacuation_failed()
+ || _cur_collection_pause_used_at_start_bytes >= cur_used_bytes,
+ "Negative collection");
+
+ size_t freed_bytes =
+ _cur_collection_pause_used_at_start_bytes - cur_used_bytes;
+ size_t surviving_bytes = _collection_set_bytes_used_before - freed_bytes;
+ double survival_fraction =
+ (double)surviving_bytes/
+ (double)_collection_set_bytes_used_before;
+
+ _n_pauses++;
+
+ if (!abandoned) {
+ _recent_CH_strong_roots_times_ms->add(_cur_CH_strong_roots_dur_ms);
+ _recent_G1_strong_roots_times_ms->add(_cur_G1_strong_roots_dur_ms);
+ _recent_evac_times_ms->add(evac_ms);
+ _recent_pause_times_ms->add(elapsed_ms);
+
+ _recent_rs_sizes->add(rs_size);
+
+ // We exempt parallel collection from this check because Alloc Buffer
+ // fragmentation can produce negative collections. Same with evac
+ // failure.
+ // Further, we're now always doing parallel collection. But I'm still
+ // leaving this here as a placeholder for a more precise assertion later.
+ // (DLD, 10/05.
+ assert((true || parallel)
+ || _g1->evacuation_failed()
+ || surviving_bytes <= _collection_set_bytes_used_before,
+ "Or else negative collection!");
+ _recent_CS_bytes_used_before->add(_collection_set_bytes_used_before);
+ _recent_CS_bytes_surviving->add(surviving_bytes);
+
+ // this is where we update the allocation rate of the application
+ double app_time_ms =
+ (_cur_collection_start_sec * 1000.0 - _prev_collection_pause_end_ms);
+ if (app_time_ms < MIN_TIMER_GRANULARITY) {
+ // This usually happens due to the timer not having the required
+ // granularity. Some Linuxes are the usual culprits.
+ // We'll just set it to something (arbitrarily) small.
+ app_time_ms = 1.0;
+ }
+ size_t regions_allocated =
+ (_region_num_young - _prev_region_num_young) +
+ (_region_num_tenured - _prev_region_num_tenured);
+ double alloc_rate_ms = (double) regions_allocated / app_time_ms;
+ _alloc_rate_ms_seq->add(alloc_rate_ms);
+ _prev_region_num_young = _region_num_young;
+ _prev_region_num_tenured = _region_num_tenured;
+
+ double interval_ms =
+ (end_time_sec - _recent_prev_end_times_for_all_gcs_sec->oldest()) * 1000.0;
+ update_recent_gc_times(end_time_sec, elapsed_ms);
+ _recent_avg_pause_time_ratio = _recent_gc_times_ms->sum()/interval_ms;
+ assert(recent_avg_pause_time_ratio() < 1.00, "All GC?");
+ }
+
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print_cr(" Recording collection pause(%d)", _n_pauses);
+ }
+
+ PauseSummary* summary;
+ if (!abandoned && !popular)
+ summary = _non_pop_summary;
+ else if (!abandoned && popular)
+ summary = _pop_summary;
+ else if (abandoned && !popular)
+ summary = _non_pop_abandoned_summary;
+ else if (abandoned && popular)
+ summary = _pop_abandoned_summary;
+ else
+ guarantee(false, "should not get here!");
+
+ double pop_update_rs_time;
+ double pop_update_rs_processed_buffers;
+ double pop_scan_rs_time;
+ double pop_closure_app_time;
+ double pop_other_time;
+
+ if (popular) {
+ PopPreambleSummary* preamble_summary = summary->pop_preamble_summary();
+ guarantee(preamble_summary != NULL, "should not be null!");
+
+ pop_update_rs_time = avg_value(_pop_par_last_update_rs_times_ms);
+ pop_update_rs_processed_buffers =
+ sum_of_values(_pop_par_last_update_rs_processed_buffers);
+ pop_scan_rs_time = avg_value(_pop_par_last_scan_rs_times_ms);
+ pop_closure_app_time = avg_value(_pop_par_last_closure_app_times_ms);
+ pop_other_time = _cur_popular_preamble_time_ms -
+ (pop_update_rs_time + pop_scan_rs_time + pop_closure_app_time +
+ _cur_popular_evac_time_ms);
+
+ preamble_summary->record_pop_preamble_time_ms(_cur_popular_preamble_time_ms);
+ preamble_summary->record_pop_update_rs_time_ms(pop_update_rs_time);
+ preamble_summary->record_pop_scan_rs_time_ms(pop_scan_rs_time);
+ preamble_summary->record_pop_closure_app_time_ms(pop_closure_app_time);
+ preamble_summary->record_pop_evacuation_time_ms(_cur_popular_evac_time_ms);
+ preamble_summary->record_pop_other_time_ms(pop_other_time);
+ }
+
+ double ext_root_scan_time = avg_value(_par_last_ext_root_scan_times_ms);
+ double mark_stack_scan_time = avg_value(_par_last_mark_stack_scan_times_ms);
+ double scan_only_time = avg_value(_par_last_scan_only_times_ms);
+ double scan_only_regions_scanned =
+ sum_of_values(_par_last_scan_only_regions_scanned);
+ double update_rs_time = avg_value(_par_last_update_rs_times_ms);
+ double update_rs_processed_buffers =
+ sum_of_values(_par_last_update_rs_processed_buffers);
+ double scan_rs_time = avg_value(_par_last_scan_rs_times_ms);
+ double obj_copy_time = avg_value(_par_last_obj_copy_times_ms);
+ double termination_time = avg_value(_par_last_termination_times_ms);
+
+ double parallel_other_time;
+ if (!abandoned) {
+ MainBodySummary* body_summary = summary->main_body_summary();
+ guarantee(body_summary != NULL, "should not be null!");
+
+ if (_satb_drain_time_set)
+ body_summary->record_satb_drain_time_ms(_cur_satb_drain_time_ms);
+ else
+ body_summary->record_satb_drain_time_ms(0.0);
+ body_summary->record_ext_root_scan_time_ms(ext_root_scan_time);
+ body_summary->record_mark_stack_scan_time_ms(mark_stack_scan_time);
+ body_summary->record_scan_only_time_ms(scan_only_time);
+ body_summary->record_update_rs_time_ms(update_rs_time);
+ body_summary->record_scan_rs_time_ms(scan_rs_time);
+ body_summary->record_obj_copy_time_ms(obj_copy_time);
+ if (parallel) {
+ body_summary->record_parallel_time_ms(_cur_collection_par_time_ms);
+ body_summary->record_clear_ct_time_ms(_cur_clear_ct_time_ms);
+ body_summary->record_termination_time_ms(termination_time);
+ parallel_other_time = _cur_collection_par_time_ms -
+ (update_rs_time + ext_root_scan_time + mark_stack_scan_time +
+ scan_only_time + scan_rs_time + obj_copy_time + termination_time);
+ body_summary->record_parallel_other_time_ms(parallel_other_time);
+ }
+ body_summary->record_mark_closure_time_ms(_mark_closure_time_ms);
+ }
+
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print_cr(" ET: %10.6f ms (avg: %10.6f ms)\n"
+ " CH Strong: %10.6f ms (avg: %10.6f ms)\n"
+ " G1 Strong: %10.6f ms (avg: %10.6f ms)\n"
+ " Evac: %10.6f ms (avg: %10.6f ms)\n"
+ " ET-RS: %10.6f ms (avg: %10.6f ms)\n"
+ " |RS|: " SIZE_FORMAT,
+ elapsed_ms, recent_avg_time_for_pauses_ms(),
+ _cur_CH_strong_roots_dur_ms, recent_avg_time_for_CH_strong_ms(),
+ _cur_G1_strong_roots_dur_ms, recent_avg_time_for_G1_strong_ms(),
+ evac_ms, recent_avg_time_for_evac_ms(),
+ scan_rs_time,
+ recent_avg_time_for_pauses_ms() -
+ recent_avg_time_for_G1_strong_ms(),
+ rs_size);
+
+ gclog_or_tty->print_cr(" Used at start: " SIZE_FORMAT"K"
+ " At end " SIZE_FORMAT "K\n"
+ " garbage : " SIZE_FORMAT "K"
+ " of " SIZE_FORMAT "K\n"
+ " survival : %6.2f%% (%6.2f%% avg)",
+ _cur_collection_pause_used_at_start_bytes/K,
+ _g1->used()/K, freed_bytes/K,
+ _collection_set_bytes_used_before/K,
+ survival_fraction*100.0,
+ recent_avg_survival_fraction()*100.0);
+ gclog_or_tty->print_cr(" Recent %% gc pause time: %6.2f",
+ recent_avg_pause_time_ratio() * 100.0);
+ }
+
+ double other_time_ms = elapsed_ms;
+ if (popular)
+ other_time_ms -= _cur_popular_preamble_time_ms;
+
+ if (!abandoned) {
+ if (_satb_drain_time_set)
+ other_time_ms -= _cur_satb_drain_time_ms;
+
+ if (parallel)
+ other_time_ms -= _cur_collection_par_time_ms + _cur_clear_ct_time_ms;
+ else
+ other_time_ms -=
+ update_rs_time +
+ ext_root_scan_time + mark_stack_scan_time + scan_only_time +
+ scan_rs_time + obj_copy_time;
+ }
+
+ if (PrintGCDetails) {
+ gclog_or_tty->print_cr("%s%s, %1.8lf secs]",
+ (popular && !abandoned) ? " (popular)" :
+ (!popular && abandoned) ? " (abandoned)" :
+ (popular && abandoned) ? " (popular/abandoned)" : "",
+ (last_pause_included_initial_mark) ? " (initial-mark)" : "",
+ elapsed_ms / 1000.0);
+
+ if (!abandoned) {
+ if (_satb_drain_time_set)
+ print_stats(1, "SATB Drain Time", _cur_satb_drain_time_ms);
+ if (_last_satb_drain_processed_buffers >= 0)
+ print_stats(2, "Processed Buffers", _last_satb_drain_processed_buffers);
+ }
+ if (popular)
+ print_stats(1, "Popularity Preamble", _cur_popular_preamble_time_ms);
+ if (parallel) {
+ if (popular) {
+ print_par_stats(2, "Update RS (Start)", _pop_par_last_update_rs_start_times_ms, false);
+ print_par_stats(2, "Update RS", _pop_par_last_update_rs_times_ms);
+ if (G1RSBarrierUseQueue)
+ print_par_buffers(3, "Processed Buffers",
+ _pop_par_last_update_rs_processed_buffers, true);
+ print_par_stats(2, "Scan RS", _pop_par_last_scan_rs_times_ms);
+ print_par_stats(2, "Closure app", _pop_par_last_closure_app_times_ms);
+ print_stats(2, "Evacuation", _cur_popular_evac_time_ms);
+ print_stats(2, "Other", pop_other_time);
+ }
+ if (!abandoned) {
+ print_stats(1, "Parallel Time", _cur_collection_par_time_ms);
+ if (!popular) {
+ print_par_stats(2, "Update RS (Start)", _par_last_update_rs_start_times_ms, false);
+ print_par_stats(2, "Update RS", _par_last_update_rs_times_ms);
+ if (G1RSBarrierUseQueue)
+ print_par_buffers(3, "Processed Buffers",
+ _par_last_update_rs_processed_buffers, true);
+ }
+ print_par_stats(2, "Ext Root Scanning", _par_last_ext_root_scan_times_ms);
+ print_par_stats(2, "Mark Stack Scanning", _par_last_mark_stack_scan_times_ms);
+ print_par_stats(2, "Scan-Only Scanning", _par_last_scan_only_times_ms);
+ print_par_buffers(3, "Scan-Only Regions",
+ _par_last_scan_only_regions_scanned, true);
+ print_par_stats(2, "Scan RS", _par_last_scan_rs_times_ms);
+ print_par_stats(2, "Object Copy", _par_last_obj_copy_times_ms);
+ print_par_stats(2, "Termination", _par_last_termination_times_ms);
+ print_stats(2, "Other", parallel_other_time);
+ print_stats(1, "Clear CT", _cur_clear_ct_time_ms);
+ }
+ } else {
+ if (popular) {
+ print_stats(2, "Update RS", pop_update_rs_time);
+ if (G1RSBarrierUseQueue)
+ print_stats(3, "Processed Buffers",
+ (int)pop_update_rs_processed_buffers);
+ print_stats(2, "Scan RS", pop_scan_rs_time);
+ print_stats(2, "Closure App", pop_closure_app_time);
+ print_stats(2, "Evacuation", _cur_popular_evac_time_ms);
+ print_stats(2, "Other", pop_other_time);
+ }
+ if (!abandoned) {
+ if (!popular) {
+ print_stats(1, "Update RS", update_rs_time);
+ if (G1RSBarrierUseQueue)
+ print_stats(2, "Processed Buffers",
+ (int)update_rs_processed_buffers);
+ }
+ print_stats(1, "Ext Root Scanning", ext_root_scan_time);
+ print_stats(1, "Mark Stack Scanning", mark_stack_scan_time);
+ print_stats(1, "Scan-Only Scanning", scan_only_time);
+ print_stats(1, "Scan RS", scan_rs_time);
+ print_stats(1, "Object Copying", obj_copy_time);
+ }
+ }
+ print_stats(1, "Other", other_time_ms);
+ for (int i = 0; i < _aux_num; ++i) {
+ if (_cur_aux_times_set[i]) {
+ char buffer[96];
+ sprintf(buffer, "Aux%d", i);
+ print_stats(1, buffer, _cur_aux_times_ms[i]);
+ }
+ }
+ }
+ if (PrintGCDetails)
+ gclog_or_tty->print(" [");
+ if (PrintGC || PrintGCDetails)
+ _g1->print_size_transition(gclog_or_tty,
+ _cur_collection_pause_used_at_start_bytes,
+ _g1->used(), _g1->capacity());
+ if (PrintGCDetails)
+ gclog_or_tty->print_cr("]");
+
+ _all_pause_times_ms->add(elapsed_ms);
+ summary->record_total_time_ms(elapsed_ms);
+ summary->record_other_time_ms(other_time_ms);
+ for (int i = 0; i < _aux_num; ++i)
+ if (_cur_aux_times_set[i])
+ _all_aux_times_ms[i].add(_cur_aux_times_ms[i]);
+
+ // Reset marks-between-pauses counter.
+ _n_marks_since_last_pause = 0;
+
+ // Update the efficiency-since-mark vars.
+ double proc_ms = elapsed_ms * (double) _parallel_gc_threads;
+ if (elapsed_ms < MIN_TIMER_GRANULARITY) {
+ // This usually happens due to the timer not having the required
+ // granularity. Some Linuxes are the usual culprits.
+ // We'll just set it to something (arbitrarily) small.
+ proc_ms = 1.0;
+ }
+ double cur_efficiency = (double) freed_bytes / proc_ms;
+
+ bool new_in_marking_window = _in_marking_window;
+ bool new_in_marking_window_im = false;
+ if (_should_initiate_conc_mark) {
+ new_in_marking_window = true;
+ new_in_marking_window_im = true;
+ }
+
+ if (in_young_gc_mode()) {
+ if (_last_full_young_gc) {
+ set_full_young_gcs(false);
+ _last_full_young_gc = false;
+ }
+
+ if ( !_last_young_gc_full ) {
+ if ( _should_revert_to_full_young_gcs ||
+ _known_garbage_ratio < 0.05 ||
+ (adaptive_young_list_length() &&
+ (get_gc_eff_factor() * cur_efficiency < predict_young_gc_eff())) ) {
+ set_full_young_gcs(true);
+ }
+ }
+ _should_revert_to_full_young_gcs = false;
+
+ if (_last_young_gc_full && !_during_marking)
+ _young_gc_eff_seq->add(cur_efficiency);
+ }
+
+ _short_lived_surv_rate_group->start_adding_regions();
+ // do that for any other surv rate groupsx
+
+ //
+
+ if (!popular && !abandoned) {
+ double pause_time_ms = elapsed_ms;
+
+ size_t diff = 0;
+ if (_max_pending_cards >= _pending_cards)
+ diff = _max_pending_cards - _pending_cards;
+ _pending_card_diff_seq->add((double) diff);
+
+ double cost_per_card_ms = 0.0;
+ if (_pending_cards > 0) {
+ cost_per_card_ms = update_rs_time / (double) _pending_cards;
+ _cost_per_card_ms_seq->add(cost_per_card_ms);
+ }
+
+ double cost_per_scan_only_region_ms = 0.0;
+ if (scan_only_regions_scanned > 0.0) {
+ cost_per_scan_only_region_ms =
+ scan_only_time / scan_only_regions_scanned;
+ if (_in_marking_window_im)
+ _cost_per_scan_only_region_ms_during_cm_seq->add(cost_per_scan_only_region_ms);
+ else
+ _cost_per_scan_only_region_ms_seq->add(cost_per_scan_only_region_ms);
+ }
+
+ size_t cards_scanned = _g1->cards_scanned();
+
+ double cost_per_entry_ms = 0.0;
+ if (cards_scanned > 10) {
+ cost_per_entry_ms = scan_rs_time / (double) cards_scanned;
+ if (_last_young_gc_full)
+ _cost_per_entry_ms_seq->add(cost_per_entry_ms);
+ else
+ _partially_young_cost_per_entry_ms_seq->add(cost_per_entry_ms);
+ }
+
+ if (_max_rs_lengths > 0) {
+ double cards_per_entry_ratio =
+ (double) cards_scanned / (double) _max_rs_lengths;
+ if (_last_young_gc_full)
+ _fully_young_cards_per_entry_ratio_seq->add(cards_per_entry_ratio);
+ else
+ _partially_young_cards_per_entry_ratio_seq->add(cards_per_entry_ratio);
+ }
+
+ size_t rs_length_diff = _max_rs_lengths - _recorded_rs_lengths;
+ if (rs_length_diff >= 0)
+ _rs_length_diff_seq->add((double) rs_length_diff);
+
+ size_t copied_bytes = surviving_bytes;
+ double cost_per_byte_ms = 0.0;
+ if (copied_bytes > 0) {
+ cost_per_byte_ms = obj_copy_time / (double) copied_bytes;
+ if (_in_marking_window)
+ _cost_per_byte_ms_during_cm_seq->add(cost_per_byte_ms);
+ else
+ _cost_per_byte_ms_seq->add(cost_per_byte_ms);
+ }
+
+ double all_other_time_ms = pause_time_ms -
+ (update_rs_time + scan_only_time + scan_rs_time + obj_copy_time +
+ _mark_closure_time_ms + termination_time);
+
+ double young_other_time_ms = 0.0;
+ if (_recorded_young_regions > 0) {
+ young_other_time_ms =
+ _recorded_young_cset_choice_time_ms +
+ _recorded_young_free_cset_time_ms;
+ _young_other_cost_per_region_ms_seq->add(young_other_time_ms /
+ (double) _recorded_young_regions);
+ }
+ double non_young_other_time_ms = 0.0;
+ if (_recorded_non_young_regions > 0) {
+ non_young_other_time_ms =
+ _recorded_non_young_cset_choice_time_ms +
+ _recorded_non_young_free_cset_time_ms;
+
+ _non_young_other_cost_per_region_ms_seq->add(non_young_other_time_ms /
+ (double) _recorded_non_young_regions);
+ }
+
+ double constant_other_time_ms = all_other_time_ms -
+ (young_other_time_ms + non_young_other_time_ms);
+ _constant_other_time_ms_seq->add(constant_other_time_ms);
+
+ double survival_ratio = 0.0;
+ if (_bytes_in_collection_set_before_gc > 0) {
+ survival_ratio = (double) bytes_in_to_space_during_gc() /
+ (double) _bytes_in_collection_set_before_gc;
+ }
+
+ _pending_cards_seq->add((double) _pending_cards);
+ _scanned_cards_seq->add((double) cards_scanned);
+ _rs_lengths_seq->add((double) _max_rs_lengths);
+
+ double expensive_region_limit_ms =
+ (double) G1MaxPauseTimeMS - predict_constant_other_time_ms();
+ if (expensive_region_limit_ms < 0.0) {
+ // this means that the other time was predicted to be longer than
+ // than the max pause time
+ expensive_region_limit_ms = (double) G1MaxPauseTimeMS;
+ }
+ _expensive_region_limit_ms = expensive_region_limit_ms;
+
+ if (PREDICTIONS_VERBOSE) {
+ gclog_or_tty->print_cr("");
+ gclog_or_tty->print_cr("PREDICTIONS %1.4lf %d "
+ "REGIONS %d %d %d %d "
+ "PENDING_CARDS %d %d "
+ "CARDS_SCANNED %d %d "
+ "RS_LENGTHS %d %d "
+ "SCAN_ONLY_SCAN %1.6lf %1.6lf "
+ "RS_UPDATE %1.6lf %1.6lf RS_SCAN %1.6lf %1.6lf "
+ "SURVIVAL_RATIO %1.6lf %1.6lf "
+ "OBJECT_COPY %1.6lf %1.6lf OTHER_CONSTANT %1.6lf %1.6lf "
+ "OTHER_YOUNG %1.6lf %1.6lf "
+ "OTHER_NON_YOUNG %1.6lf %1.6lf "
+ "VTIME_DIFF %1.6lf TERMINATION %1.6lf "
+ "ELAPSED %1.6lf %1.6lf ",
+ _cur_collection_start_sec,
+ (!_last_young_gc_full) ? 2 :
+ (last_pause_included_initial_mark) ? 1 : 0,
+ _recorded_region_num,
+ _recorded_young_regions,
+ _recorded_scan_only_regions,
+ _recorded_non_young_regions,
+ _predicted_pending_cards, _pending_cards,
+ _predicted_cards_scanned, cards_scanned,
+ _predicted_rs_lengths, _max_rs_lengths,
+ _predicted_scan_only_scan_time_ms, scan_only_time,
+ _predicted_rs_update_time_ms, update_rs_time,
+ _predicted_rs_scan_time_ms, scan_rs_time,
+ _predicted_survival_ratio, survival_ratio,
+ _predicted_object_copy_time_ms, obj_copy_time,
+ _predicted_constant_other_time_ms, constant_other_time_ms,
+ _predicted_young_other_time_ms, young_other_time_ms,
+ _predicted_non_young_other_time_ms,
+ non_young_other_time_ms,
+ _vtime_diff_ms, termination_time,
+ _predicted_pause_time_ms, elapsed_ms);
+ }
+
+ if (G1PolicyVerbose > 0) {
+ gclog_or_tty->print_cr("Pause Time, predicted: %1.4lfms (predicted %s), actual: %1.4lfms",
+ _predicted_pause_time_ms,
+ (_within_target) ? "within" : "outside",
+ elapsed_ms);
+ }
+
+ }
+
+ _in_marking_window = new_in_marking_window;
+ _in_marking_window_im = new_in_marking_window_im;
+ _free_regions_at_end_of_collection = _g1->free_regions();
+ _scan_only_regions_at_end_of_collection = _g1->young_list_length();
+ calculate_young_list_min_length();
+ calculate_young_list_target_config();
+
+ //
+
+ _target_pause_time_ms = -1.0;
+
+ // TODO: calculate tenuring threshold
+ _tenuring_threshold = MaxTenuringThreshold;
+}
+
+//
+
+double
+G1CollectorPolicy::
+predict_young_collection_elapsed_time_ms(size_t adjustment) {
+ guarantee( adjustment == 0 || adjustment == 1, "invariant" );
+
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ size_t young_num = g1h->young_list_length();
+ if (young_num == 0)
+ return 0.0;
+
+ young_num += adjustment;
+ size_t pending_cards = predict_pending_cards();
+ size_t rs_lengths = g1h->young_list_sampled_rs_lengths() +
+ predict_rs_length_diff();
+ size_t card_num;
+ if (full_young_gcs())
+ card_num = predict_young_card_num(rs_lengths);
+ else
+ card_num = predict_non_young_card_num(rs_lengths);
+ size_t young_byte_size = young_num * HeapRegion::GrainBytes;
+ double accum_yg_surv_rate =
+ _short_lived_surv_rate_group->accum_surv_rate(adjustment);
+
+ size_t bytes_to_copy =
+ (size_t) (accum_yg_surv_rate * (double) HeapRegion::GrainBytes);
+
+ return
+ predict_rs_update_time_ms(pending_cards) +
+ predict_rs_scan_time_ms(card_num) +
+ predict_object_copy_time_ms(bytes_to_copy) +
+ predict_young_other_time_ms(young_num) +
+ predict_constant_other_time_ms();
+}
+
+double
+G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards) {
+ size_t rs_length = predict_rs_length_diff();
+ size_t card_num;
+ if (full_young_gcs())
+ card_num = predict_young_card_num(rs_length);
+ else
+ card_num = predict_non_young_card_num(rs_length);
+ return predict_base_elapsed_time_ms(pending_cards, card_num);
+}
+
+double
+G1CollectorPolicy::predict_base_elapsed_time_ms(size_t pending_cards,
+ size_t scanned_cards) {
+ return
+ predict_rs_update_time_ms(pending_cards) +
+ predict_rs_scan_time_ms(scanned_cards) +
+ predict_constant_other_time_ms();
+}
+
+double
+G1CollectorPolicy::predict_region_elapsed_time_ms(HeapRegion* hr,
+ bool young) {
+ size_t rs_length = hr->rem_set()->occupied();
+ size_t card_num;
+ if (full_young_gcs())
+ card_num = predict_young_card_num(rs_length);
+ else
+ card_num = predict_non_young_card_num(rs_length);
+ size_t bytes_to_copy = predict_bytes_to_copy(hr);
+
+ double region_elapsed_time_ms =
+ predict_rs_scan_time_ms(card_num) +
+ predict_object_copy_time_ms(bytes_to_copy);
+
+ if (young)
+ region_elapsed_time_ms += predict_young_other_time_ms(1);
+ else
+ region_elapsed_time_ms += predict_non_young_other_time_ms(1);
+
+ return region_elapsed_time_ms;
+}
+
+size_t
+G1CollectorPolicy::predict_bytes_to_copy(HeapRegion* hr) {
+ size_t bytes_to_copy;
+ if (hr->is_marked())
+ bytes_to_copy = hr->max_live_bytes();
+ else {
+ guarantee( hr->is_young() && hr->age_in_surv_rate_group() != -1,
+ "invariant" );
+ int age = hr->age_in_surv_rate_group();
+ double yg_surv_rate = predict_yg_surv_rate(age);
+ bytes_to_copy = (size_t) ((double) hr->used() * yg_surv_rate);
+ }
+
+ return bytes_to_copy;
+}
+
+void
+G1CollectorPolicy::start_recording_regions() {
+ _recorded_rs_lengths = 0;
+ _recorded_scan_only_regions = 0;
+ _recorded_young_regions = 0;
+ _recorded_non_young_regions = 0;
+
+#if PREDICTIONS_VERBOSE
+ _predicted_rs_lengths = 0;
+ _predicted_cards_scanned = 0;
+
+ _recorded_marked_bytes = 0;
+ _recorded_young_bytes = 0;
+ _predicted_bytes_to_copy = 0;
+#endif // PREDICTIONS_VERBOSE
+}
+
+void
+G1CollectorPolicy::record_cset_region(HeapRegion* hr, bool young) {
+ if (young) {
+ ++_recorded_young_regions;
+ } else {
+ ++_recorded_non_young_regions;
+ }
+#if PREDICTIONS_VERBOSE
+ if (young) {
+ _recorded_young_bytes += hr->asSpace()->used();
+ } else {
+ _recorded_marked_bytes += hr->max_live_bytes();
+ }
+ _predicted_bytes_to_copy += predict_bytes_to_copy(hr);
+#endif // PREDICTIONS_VERBOSE
+
+ size_t rs_length = hr->rem_set()->occupied();
+ _recorded_rs_lengths += rs_length;
+}
+
+void
+G1CollectorPolicy::record_scan_only_regions(size_t scan_only_length) {
+ _recorded_scan_only_regions = scan_only_length;
+}
+
+void
+G1CollectorPolicy::end_recording_regions() {
+#if PREDICTIONS_VERBOSE
+ _predicted_pending_cards = predict_pending_cards();
+ _predicted_rs_lengths = _recorded_rs_lengths + predict_rs_length_diff();
+ if (full_young_gcs())
+ _predicted_cards_scanned += predict_young_card_num(_predicted_rs_lengths);
+ else
+ _predicted_cards_scanned +=
+ predict_non_young_card_num(_predicted_rs_lengths);
+ _recorded_region_num = _recorded_young_regions + _recorded_non_young_regions;
+
+ _predicted_young_survival_ratio = 0.0;
+ for (int i = 0; i < _recorded_young_regions; ++i)
+ _predicted_young_survival_ratio += predict_yg_surv_rate(i);
+ _predicted_young_survival_ratio /= (double) _recorded_young_regions;
+
+ _predicted_scan_only_scan_time_ms =
+ predict_scan_only_time_ms(_recorded_scan_only_regions);
+ _predicted_rs_update_time_ms =
+ predict_rs_update_time_ms(_g1->pending_card_num());
+ _predicted_rs_scan_time_ms =
+ predict_rs_scan_time_ms(_predicted_cards_scanned);
+ _predicted_object_copy_time_ms =
+ predict_object_copy_time_ms(_predicted_bytes_to_copy);
+ _predicted_constant_other_time_ms =
+ predict_constant_other_time_ms();
+ _predicted_young_other_time_ms =
+ predict_young_other_time_ms(_recorded_young_regions);
+ _predicted_non_young_other_time_ms =
+ predict_non_young_other_time_ms(_recorded_non_young_regions);
+
+ _predicted_pause_time_ms =
+ _predicted_scan_only_scan_time_ms +
+ _predicted_rs_update_time_ms +
+ _predicted_rs_scan_time_ms +
+ _predicted_object_copy_time_ms +
+ _predicted_constant_other_time_ms +
+ _predicted_young_other_time_ms +
+ _predicted_non_young_other_time_ms;
+#endif // PREDICTIONS_VERBOSE
+}
+
+void G1CollectorPolicy::check_if_region_is_too_expensive(double
+ predicted_time_ms) {
+ // I don't think we need to do this when in young GC mode since
+ // marking will be initiated next time we hit the soft limit anyway...
+ if (predicted_time_ms > _expensive_region_limit_ms) {
+ if (!in_young_gc_mode()) {
+ set_full_young_gcs(true);
+ _should_initiate_conc_mark = true;
+ } else
+ // no point in doing another partial one
+ _should_revert_to_full_young_gcs = true;
+ }
+}
+
+//
+
+
+void G1CollectorPolicy::update_recent_gc_times(double end_time_sec,
+ double elapsed_ms) {
+ _recent_gc_times_ms->add(elapsed_ms);
+ _recent_prev_end_times_for_all_gcs_sec->add(end_time_sec);
+ _prev_collection_pause_end_ms = end_time_sec * 1000.0;
+}
+
+double G1CollectorPolicy::recent_avg_time_for_pauses_ms() {
+ if (_recent_pause_times_ms->num() == 0) return (double) G1MaxPauseTimeMS;
+ else return _recent_pause_times_ms->avg();
+}
+
+double G1CollectorPolicy::recent_avg_time_for_CH_strong_ms() {
+ if (_recent_CH_strong_roots_times_ms->num() == 0)
+ return (double)G1MaxPauseTimeMS/3.0;
+ else return _recent_CH_strong_roots_times_ms->avg();
+}
+
+double G1CollectorPolicy::recent_avg_time_for_G1_strong_ms() {
+ if (_recent_G1_strong_roots_times_ms->num() == 0)
+ return (double)G1MaxPauseTimeMS/3.0;
+ else return _recent_G1_strong_roots_times_ms->avg();
+}
+
+double G1CollectorPolicy::recent_avg_time_for_evac_ms() {
+ if (_recent_evac_times_ms->num() == 0) return (double)G1MaxPauseTimeMS/3.0;
+ else return _recent_evac_times_ms->avg();
+}
+
+int G1CollectorPolicy::number_of_recent_gcs() {
+ assert(_recent_CH_strong_roots_times_ms->num() ==
+ _recent_G1_strong_roots_times_ms->num(), "Sequence out of sync");
+ assert(_recent_G1_strong_roots_times_ms->num() ==
+ _recent_evac_times_ms->num(), "Sequence out of sync");
+ assert(_recent_evac_times_ms->num() ==
+ _recent_pause_times_ms->num(), "Sequence out of sync");
+ assert(_recent_pause_times_ms->num() ==
+ _recent_CS_bytes_used_before->num(), "Sequence out of sync");
+ assert(_recent_CS_bytes_used_before->num() ==
+ _recent_CS_bytes_surviving->num(), "Sequence out of sync");
+ return _recent_pause_times_ms->num();
+}
+
+double G1CollectorPolicy::recent_avg_survival_fraction() {
+ return recent_avg_survival_fraction_work(_recent_CS_bytes_surviving,
+ _recent_CS_bytes_used_before);
+}
+
+double G1CollectorPolicy::last_survival_fraction() {
+ return last_survival_fraction_work(_recent_CS_bytes_surviving,
+ _recent_CS_bytes_used_before);
+}
+
+double
+G1CollectorPolicy::recent_avg_survival_fraction_work(TruncatedSeq* surviving,
+ TruncatedSeq* before) {
+ assert(surviving->num() == before->num(), "Sequence out of sync");
+ if (before->sum() > 0.0) {
+ double recent_survival_rate = surviving->sum() / before->sum();
+ // We exempt parallel collection from this check because Alloc Buffer
+ // fragmentation can produce negative collections.
+ // Further, we're now always doing parallel collection. But I'm still
+ // leaving this here as a placeholder for a more precise assertion later.
+ // (DLD, 10/05.)
+ assert((true || ParallelGCThreads > 0) ||
+ _g1->evacuation_failed() ||
+ recent_survival_rate <= 1.0, "Or bad frac");
+ return recent_survival_rate;
+ } else {
+ return 1.0; // Be conservative.
+ }
+}
+
+double
+G1CollectorPolicy::last_survival_fraction_work(TruncatedSeq* surviving,
+ TruncatedSeq* before) {
+ assert(surviving->num() == before->num(), "Sequence out of sync");
+ if (surviving->num() > 0 && before->last() > 0.0) {
+ double last_survival_rate = surviving->last() / before->last();
+ // We exempt parallel collection from this check because Alloc Buffer
+ // fragmentation can produce negative collections.
+ // Further, we're now always doing parallel collection. But I'm still
+ // leaving this here as a placeholder for a more precise assertion later.
+ // (DLD, 10/05.)
+ assert((true || ParallelGCThreads > 0) ||
+ last_survival_rate <= 1.0, "Or bad frac");
+ return last_survival_rate;
+ } else {
+ return 1.0;
+ }
+}
+
+static const int survival_min_obs = 5;
+static double survival_min_obs_limits[] = { 0.9, 0.7, 0.5, 0.3, 0.1 };
+static const double min_survival_rate = 0.1;
+
+double
+G1CollectorPolicy::conservative_avg_survival_fraction_work(double avg,
+ double latest) {
+ double res = avg;
+ if (number_of_recent_gcs() < survival_min_obs) {
+ res = MAX2(res, survival_min_obs_limits[number_of_recent_gcs()]);
+ }
+ res = MAX2(res, latest);
+ res = MAX2(res, min_survival_rate);
+ // In the parallel case, LAB fragmentation can produce "negative
+ // collections"; so can evac failure. Cap at 1.0
+ res = MIN2(res, 1.0);
+ return res;
+}
+
+size_t G1CollectorPolicy::expansion_amount() {
+ if ((int)(recent_avg_pause_time_ratio() * 100.0) > G1GCPct) {
+ // We will double the existing space, or take G1ExpandByPctOfAvail % of
+ // the available expansion space, whichever is smaller, bounded below
+ // by a minimum expansion (unless that's all that's left.)
+ const size_t min_expand_bytes = 1*M;
+ size_t reserved_bytes = _g1->g1_reserved_obj_bytes();
+ size_t committed_bytes = _g1->capacity();
+ size_t uncommitted_bytes = reserved_bytes - committed_bytes;
+ size_t expand_bytes;
+ size_t expand_bytes_via_pct =
+ uncommitted_bytes * G1ExpandByPctOfAvail / 100;
+ expand_bytes = MIN2(expand_bytes_via_pct, committed_bytes);
+ expand_bytes = MAX2(expand_bytes, min_expand_bytes);
+ expand_bytes = MIN2(expand_bytes, uncommitted_bytes);
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print("Decided to expand: ratio = %5.2f, "
+ "committed = %d%s, uncommited = %d%s, via pct = %d%s.\n"
+ " Answer = %d.\n",
+ recent_avg_pause_time_ratio(),
+ byte_size_in_proper_unit(committed_bytes),
+ proper_unit_for_byte_size(committed_bytes),
+ byte_size_in_proper_unit(uncommitted_bytes),
+ proper_unit_for_byte_size(uncommitted_bytes),
+ byte_size_in_proper_unit(expand_bytes_via_pct),
+ proper_unit_for_byte_size(expand_bytes_via_pct),
+ byte_size_in_proper_unit(expand_bytes),
+ proper_unit_for_byte_size(expand_bytes));
+ }
+ return expand_bytes;
+ } else {
+ return 0;
+ }
+}
+
+void G1CollectorPolicy::note_start_of_mark_thread() {
+ _mark_thread_startup_sec = os::elapsedTime();
+}
+
+class CountCSClosure: public HeapRegionClosure {
+ G1CollectorPolicy* _g1_policy;
+public:
+ CountCSClosure(G1CollectorPolicy* g1_policy) :
+ _g1_policy(g1_policy) {}
+ bool doHeapRegion(HeapRegion* r) {
+ _g1_policy->_bytes_in_collection_set_before_gc += r->used();
+ return false;
+ }
+};
+
+void G1CollectorPolicy::count_CS_bytes_used() {
+ CountCSClosure cs_closure(this);
+ _g1->collection_set_iterate(&cs_closure);
+}
+
+static void print_indent(int level) {
+ for (int j = 0; j < level+1; ++j)
+ gclog_or_tty->print(" ");
+}
+
+void G1CollectorPolicy::print_summary (int level,
+ const char* str,
+ NumberSeq* seq) const {
+ double sum = seq->sum();
+ print_indent(level);
+ gclog_or_tty->print_cr("%-24s = %8.2lf s (avg = %8.2lf ms)",
+ str, sum / 1000.0, seq->avg());
+}
+
+void G1CollectorPolicy::print_summary_sd (int level,
+ const char* str,
+ NumberSeq* seq) const {
+ print_summary(level, str, seq);
+ print_indent(level + 5);
+ gclog_or_tty->print_cr("(num = %5d, std dev = %8.2lf ms, max = %8.2lf ms)",
+ seq->num(), seq->sd(), seq->maximum());
+}
+
+void G1CollectorPolicy::check_other_times(int level,
+ NumberSeq* other_times_ms,
+ NumberSeq* calc_other_times_ms) const {
+ bool should_print = false;
+
+ double max_sum = MAX2(fabs(other_times_ms->sum()),
+ fabs(calc_other_times_ms->sum()));
+ double min_sum = MIN2(fabs(other_times_ms->sum()),
+ fabs(calc_other_times_ms->sum()));
+ double sum_ratio = max_sum / min_sum;
+ if (sum_ratio > 1.1) {
+ should_print = true;
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## CALCULATED OTHER SUM DOESN'T MATCH RECORDED ###");
+ }
+
+ double max_avg = MAX2(fabs(other_times_ms->avg()),
+ fabs(calc_other_times_ms->avg()));
+ double min_avg = MIN2(fabs(other_times_ms->avg()),
+ fabs(calc_other_times_ms->avg()));
+ double avg_ratio = max_avg / min_avg;
+ if (avg_ratio > 1.1) {
+ should_print = true;
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## CALCULATED OTHER AVG DOESN'T MATCH RECORDED ###");
+ }
+
+ if (other_times_ms->sum() < -0.01) {
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## RECORDED OTHER SUM IS NEGATIVE ###");
+ }
+
+ if (other_times_ms->avg() < -0.01) {
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## RECORDED OTHER AVG IS NEGATIVE ###");
+ }
+
+ if (calc_other_times_ms->sum() < -0.01) {
+ should_print = true;
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## CALCULATED OTHER SUM IS NEGATIVE ###");
+ }
+
+ if (calc_other_times_ms->avg() < -0.01) {
+ should_print = true;
+ print_indent(level + 1);
+ gclog_or_tty->print_cr("## CALCULATED OTHER AVG IS NEGATIVE ###");
+ }
+
+ if (should_print)
+ print_summary(level, "Other(Calc)", calc_other_times_ms);
+}
+
+void G1CollectorPolicy::print_summary(PauseSummary* summary) const {
+ bool parallel = ParallelGCThreads > 0;
+ MainBodySummary* body_summary = summary->main_body_summary();
+ PopPreambleSummary* preamble_summary = summary->pop_preamble_summary();
+
+ if (summary->get_total_seq()->num() > 0) {
+ print_summary_sd(0,
+ (preamble_summary == NULL) ? "Non-Popular Pauses" :
+ "Popular Pauses",
+ summary->get_total_seq());
+ if (preamble_summary != NULL) {
+ print_summary(1, "Popularity Preamble",
+ preamble_summary->get_pop_preamble_seq());
+ print_summary(2, "Update RS", preamble_summary->get_pop_update_rs_seq());
+ print_summary(2, "Scan RS", preamble_summary->get_pop_scan_rs_seq());
+ print_summary(2, "Closure App",
+ preamble_summary->get_pop_closure_app_seq());
+ print_summary(2, "Evacuation",
+ preamble_summary->get_pop_evacuation_seq());
+ print_summary(2, "Other", preamble_summary->get_pop_other_seq());
+ {
+ NumberSeq* other_parts[] = {
+ preamble_summary->get_pop_update_rs_seq(),
+ preamble_summary->get_pop_scan_rs_seq(),
+ preamble_summary->get_pop_closure_app_seq(),
+ preamble_summary->get_pop_evacuation_seq()
+ };
+ NumberSeq calc_other_times_ms(preamble_summary->get_pop_preamble_seq(),
+ 4, other_parts);
+ check_other_times(2, preamble_summary->get_pop_other_seq(),
+ &calc_other_times_ms);
+ }
+ }
+ if (body_summary != NULL) {
+ print_summary(1, "SATB Drain", body_summary->get_satb_drain_seq());
+ if (parallel) {
+ print_summary(1, "Parallel Time", body_summary->get_parallel_seq());
+ print_summary(2, "Update RS", body_summary->get_update_rs_seq());
+ print_summary(2, "Ext Root Scanning",
+ body_summary->get_ext_root_scan_seq());
+ print_summary(2, "Mark Stack Scanning",
+ body_summary->get_mark_stack_scan_seq());
+ print_summary(2, "Scan-Only Scanning",
+ body_summary->get_scan_only_seq());
+ print_summary(2, "Scan RS", body_summary->get_scan_rs_seq());
+ print_summary(2, "Object Copy", body_summary->get_obj_copy_seq());
+ print_summary(2, "Termination", body_summary->get_termination_seq());
+ print_summary(2, "Other", body_summary->get_parallel_other_seq());
+ {
+ NumberSeq* other_parts[] = {
+ body_summary->get_update_rs_seq(),
+ body_summary->get_ext_root_scan_seq(),
+ body_summary->get_mark_stack_scan_seq(),
+ body_summary->get_scan_only_seq(),
+ body_summary->get_scan_rs_seq(),
+ body_summary->get_obj_copy_seq(),
+ body_summary->get_termination_seq()
+ };
+ NumberSeq calc_other_times_ms(body_summary->get_parallel_seq(),
+ 7, other_parts);
+ check_other_times(2, body_summary->get_parallel_other_seq(),
+ &calc_other_times_ms);
+ }
+ print_summary(1, "Mark Closure", body_summary->get_mark_closure_seq());
+ print_summary(1, "Clear CT", body_summary->get_clear_ct_seq());
+ } else {
+ print_summary(1, "Update RS", body_summary->get_update_rs_seq());
+ print_summary(1, "Ext Root Scanning",
+ body_summary->get_ext_root_scan_seq());
+ print_summary(1, "Mark Stack Scanning",
+ body_summary->get_mark_stack_scan_seq());
+ print_summary(1, "Scan-Only Scanning",
+ body_summary->get_scan_only_seq());
+ print_summary(1, "Scan RS", body_summary->get_scan_rs_seq());
+ print_summary(1, "Object Copy", body_summary->get_obj_copy_seq());
+ }
+ }
+ print_summary(1, "Other", summary->get_other_seq());
+ {
+ NumberSeq calc_other_times_ms;
+ if (body_summary != NULL) {
+ // not abandoned
+ if (parallel) {
+ // parallel
+ NumberSeq* other_parts[] = {
+ body_summary->get_satb_drain_seq(),
+ (preamble_summary == NULL) ? NULL :
+ preamble_summary->get_pop_preamble_seq(),
+ body_summary->get_parallel_seq(),
+ body_summary->get_clear_ct_seq()
+ };
+ calc_other_times_ms = NumberSeq (summary->get_total_seq(),
+ 4, other_parts);
+ } else {
+ // serial
+ NumberSeq* other_parts[] = {
+ body_summary->get_satb_drain_seq(),
+ (preamble_summary == NULL) ? NULL :
+ preamble_summary->get_pop_preamble_seq(),
+ body_summary->get_update_rs_seq(),
+ body_summary->get_ext_root_scan_seq(),
+ body_summary->get_mark_stack_scan_seq(),
+ body_summary->get_scan_only_seq(),
+ body_summary->get_scan_rs_seq(),
+ body_summary->get_obj_copy_seq()
+ };
+ calc_other_times_ms = NumberSeq(summary->get_total_seq(),
+ 8, other_parts);
+ }
+ } else {
+ // abandoned
+ NumberSeq* other_parts[] = {
+ (preamble_summary == NULL) ? NULL :
+ preamble_summary->get_pop_preamble_seq()
+ };
+ calc_other_times_ms = NumberSeq(summary->get_total_seq(),
+ 1, other_parts);
+ }
+ check_other_times(1, summary->get_other_seq(), &calc_other_times_ms);
+ }
+ } else {
+ print_indent(0);
+ gclog_or_tty->print_cr("none");
+ }
+ gclog_or_tty->print_cr("");
+}
+
+void
+G1CollectorPolicy::print_abandoned_summary(PauseSummary* non_pop_summary,
+ PauseSummary* pop_summary) const {
+ bool printed = false;
+ if (non_pop_summary->get_total_seq()->num() > 0) {
+ printed = true;
+ print_summary(non_pop_summary);
+ }
+ if (pop_summary->get_total_seq()->num() > 0) {
+ printed = true;
+ print_summary(pop_summary);
+ }
+
+ if (!printed) {
+ print_indent(0);
+ gclog_or_tty->print_cr("none");
+ gclog_or_tty->print_cr("");
+ }
+}
+
+void G1CollectorPolicy::print_tracing_info() const {
+ if (TraceGen0Time) {
+ gclog_or_tty->print_cr("ALL PAUSES");
+ print_summary_sd(0, "Total", _all_pause_times_ms);
+ gclog_or_tty->print_cr("");
+ gclog_or_tty->print_cr("");
+ gclog_or_tty->print_cr(" Full Young GC Pauses: %8d", _full_young_pause_num);
+ gclog_or_tty->print_cr(" Partial Young GC Pauses: %8d", _partial_young_pause_num);
+ gclog_or_tty->print_cr("");
+
+ gclog_or_tty->print_cr("NON-POPULAR PAUSES");
+ print_summary(_non_pop_summary);
+
+ gclog_or_tty->print_cr("POPULAR PAUSES");
+ print_summary(_pop_summary);
+
+ gclog_or_tty->print_cr("ABANDONED PAUSES");
+ print_abandoned_summary(_non_pop_abandoned_summary,
+ _pop_abandoned_summary);
+
+ gclog_or_tty->print_cr("MISC");
+ print_summary_sd(0, "Stop World", _all_stop_world_times_ms);
+ print_summary_sd(0, "Yields", _all_yield_times_ms);
+ for (int i = 0; i < _aux_num; ++i) {
+ if (_all_aux_times_ms[i].num() > 0) {
+ char buffer[96];
+ sprintf(buffer, "Aux%d", i);
+ print_summary_sd(0, buffer, &_all_aux_times_ms[i]);
+ }
+ }
+
+ size_t all_region_num = _region_num_young + _region_num_tenured;
+ gclog_or_tty->print_cr(" New Regions %8d, Young %8d (%6.2lf%%), "
+ "Tenured %8d (%6.2lf%%)",
+ all_region_num,
+ _region_num_young,
+ (double) _region_num_young / (double) all_region_num * 100.0,
+ _region_num_tenured,
+ (double) _region_num_tenured / (double) all_region_num * 100.0);
+
+ if (!G1RSBarrierUseQueue) {
+ gclog_or_tty->print_cr("Of %d times conc refinement was enabled, %d (%7.2f%%) "
+ "did zero traversals.",
+ _conc_refine_enabled, _conc_refine_zero_traversals,
+ _conc_refine_enabled > 0 ?
+ 100.0 * (float)_conc_refine_zero_traversals/
+ (float)_conc_refine_enabled : 0.0);
+ gclog_or_tty->print_cr(" Max # of traversals = %d.",
+ _conc_refine_max_traversals);
+ gclog_or_tty->print_cr("");
+ }
+ }
+ if (TraceGen1Time) {
+ if (_all_full_gc_times_ms->num() > 0) {
+ gclog_or_tty->print("\n%4d full_gcs: total time = %8.2f s",
+ _all_full_gc_times_ms->num(),
+ _all_full_gc_times_ms->sum() / 1000.0);
+ gclog_or_tty->print_cr(" (avg = %8.2fms).", _all_full_gc_times_ms->avg());
+ gclog_or_tty->print_cr(" [std. dev = %8.2f ms, max = %8.2f ms]",
+ _all_full_gc_times_ms->sd(),
+ _all_full_gc_times_ms->maximum());
+ }
+ }
+}
+
+void G1CollectorPolicy::print_yg_surv_rate_info() const {
+#ifndef PRODUCT
+ _short_lived_surv_rate_group->print_surv_rate_summary();
+ // add this call for any other surv rate groups
+#endif // PRODUCT
+}
+
+void G1CollectorPolicy::update_conc_refine_data() {
+ unsigned traversals = _g1->concurrent_g1_refine()->disable();
+ if (traversals == 0) _conc_refine_zero_traversals++;
+ _conc_refine_max_traversals = MAX2(_conc_refine_max_traversals,
+ (size_t)traversals);
+
+ if (G1PolicyVerbose > 1)
+ gclog_or_tty->print_cr("Did a CR traversal series: %d traversals.", traversals);
+ double multiplier = 1.0;
+ if (traversals == 0) {
+ multiplier = 4.0;
+ } else if (traversals > (size_t)G1ConcRefineTargTraversals) {
+ multiplier = 1.0/1.5;
+ } else if (traversals < (size_t)G1ConcRefineTargTraversals) {
+ multiplier = 1.5;
+ }
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print_cr(" Multiplier = %7.2f.", multiplier);
+ gclog_or_tty->print(" Delta went from %d regions to ",
+ _conc_refine_current_delta);
+ }
+ _conc_refine_current_delta =
+ MIN2(_g1->n_regions(),
+ (size_t)(_conc_refine_current_delta * multiplier));
+ _conc_refine_current_delta =
+ MAX2(_conc_refine_current_delta, (size_t)1);
+ if (G1PolicyVerbose > 1) {
+ gclog_or_tty->print_cr("%d regions.", _conc_refine_current_delta);
+ }
+ _conc_refine_enabled++;
+}
+
+void G1CollectorPolicy::set_single_region_collection_set(HeapRegion* hr) {
+ assert(collection_set() == NULL, "Must be no current CS.");
+ _collection_set_size = 0;
+ _collection_set_bytes_used_before = 0;
+ add_to_collection_set(hr);
+ count_CS_bytes_used();
+}
+
+bool
+G1CollectorPolicy::should_add_next_region_to_young_list() {
+ assert(in_young_gc_mode(), "should be in young GC mode");
+ bool ret;
+ size_t young_list_length = _g1->young_list_length();
+
+ if (young_list_length < _young_list_target_length) {
+ ret = true;
+ ++_region_num_young;
+ } else {
+ ret = false;
+ ++_region_num_tenured;
+ }
+
+ return ret;
+}
+
+#ifndef PRODUCT
+// for debugging, bit of a hack...
+static char*
+region_num_to_mbs(int length) {
+ static char buffer[64];
+ double bytes = (double) (length * HeapRegion::GrainBytes);
+ double mbs = bytes / (double) (1024 * 1024);
+ sprintf(buffer, "%7.2lfMB", mbs);
+ return buffer;
+}
+#endif // PRODUCT
+
+void
+G1CollectorPolicy::checkpoint_conc_overhead() {
+ double conc_overhead = 0.0;
+ if (G1AccountConcurrentOverhead)
+ conc_overhead = COTracker::totalPredConcOverhead();
+ _mmu_tracker->update_conc_overhead(conc_overhead);
+#if 0
+ gclog_or_tty->print(" CO %1.4lf TARGET %1.4lf",
+ conc_overhead, _mmu_tracker->max_gc_time());
+#endif
+}
+
+
+uint G1CollectorPolicy::max_regions(int purpose) {
+ switch (purpose) {
+ case GCAllocForSurvived:
+ return G1MaxSurvivorRegions;
+ case GCAllocForTenured:
+ return UINT_MAX;
+ default:
+ return UINT_MAX;
+ };
+}
+
+void
+G1CollectorPolicy_BestRegionsFirst::
+set_single_region_collection_set(HeapRegion* hr) {
+ G1CollectorPolicy::set_single_region_collection_set(hr);
+ _collectionSetChooser->removeRegion(hr);
+}
+
+
+bool
+G1CollectorPolicy_BestRegionsFirst::should_do_collection_pause(size_t
+ word_size) {
+ assert(_g1->regions_accounted_for(), "Region leakage!");
+ // Initiate a pause when we reach the steady-state "used" target.
+ size_t used_hard = (_g1->capacity() / 100) * G1SteadyStateUsed;
+ size_t used_soft =
+ MAX2((_g1->capacity() / 100) * (G1SteadyStateUsed - G1SteadyStateUsedDelta),
+ used_hard/2);
+ size_t used = _g1->used();
+
+ double max_pause_time_ms = _mmu_tracker->max_gc_time() * 1000.0;
+
+ size_t young_list_length = _g1->young_list_length();
+ bool reached_target_length = young_list_length >= _young_list_target_length;
+
+ if (in_young_gc_mode()) {
+ if (reached_target_length) {
+ assert( young_list_length > 0 && _g1->young_list_length() > 0,
+ "invariant" );
+ _target_pause_time_ms = max_pause_time_ms;
+ return true;
+ }
+ } else {
+ guarantee( false, "should not reach here" );
+ }
+
+ return false;
+}
+
+#ifndef PRODUCT
+class HRSortIndexIsOKClosure: public HeapRegionClosure {
+ CollectionSetChooser* _chooser;
+public:
+ HRSortIndexIsOKClosure(CollectionSetChooser* chooser) :
+ _chooser(chooser) {}
+
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->continuesHumongous()) {
+ assert(_chooser->regionProperlyOrdered(r), "Ought to be.");
+ }
+ return false;
+ }
+};
+
+bool G1CollectorPolicy_BestRegionsFirst::assertMarkedBytesDataOK() {
+ HRSortIndexIsOKClosure cl(_collectionSetChooser);
+ _g1->heap_region_iterate(&cl);
+ return true;
+}
+#endif
+
+void
+G1CollectorPolicy_BestRegionsFirst::
+record_collection_pause_start(double start_time_sec, size_t start_used) {
+ G1CollectorPolicy::record_collection_pause_start(start_time_sec, start_used);
+}
+
+class NextNonCSElemFinder: public HeapRegionClosure {
+ HeapRegion* _res;
+public:
+ NextNonCSElemFinder(): _res(NULL) {}
+ bool doHeapRegion(HeapRegion* r) {
+ if (!r->in_collection_set()) {
+ _res = r;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ HeapRegion* res() { return _res; }
+};
+
+class KnownGarbageClosure: public HeapRegionClosure {
+ CollectionSetChooser* _hrSorted;
+
+public:
+ KnownGarbageClosure(CollectionSetChooser* hrSorted) :
+ _hrSorted(hrSorted)
+ {}
+
+ bool doHeapRegion(HeapRegion* r) {
+ // We only include humongous regions in collection
+ // sets when concurrent mark shows that their contained object is
+ // unreachable.
+
+ // Do we have any marking information for this region?
+ if (r->is_marked()) {
+ // We don't include humongous regions in collection
+ // sets because we collect them immediately at the end of a marking
+ // cycle. We also don't include young regions because we *must*
+ // include them in the next collection pause.
+ if (!r->isHumongous() && !r->is_young()) {
+ _hrSorted->addMarkedHeapRegion(r);
+ }
+ }
+ return false;
+ }
+};
+
+class ParKnownGarbageHRClosure: public HeapRegionClosure {
+ CollectionSetChooser* _hrSorted;
+ jint _marked_regions_added;
+ jint _chunk_size;
+ jint _cur_chunk_idx;
+ jint _cur_chunk_end; // Cur chunk [_cur_chunk_idx, _cur_chunk_end)
+ int _worker;
+ int _invokes;
+
+ void get_new_chunk() {
+ _cur_chunk_idx = _hrSorted->getParMarkedHeapRegionChunk(_chunk_size);
+ _cur_chunk_end = _cur_chunk_idx + _chunk_size;
+ }
+ void add_region(HeapRegion* r) {
+ if (_cur_chunk_idx == _cur_chunk_end) {
+ get_new_chunk();
+ }
+ assert(_cur_chunk_idx < _cur_chunk_end, "postcondition");
+ _hrSorted->setMarkedHeapRegion(_cur_chunk_idx, r);
+ _marked_regions_added++;
+ _cur_chunk_idx++;
+ }
+
+public:
+ ParKnownGarbageHRClosure(CollectionSetChooser* hrSorted,
+ jint chunk_size,
+ int worker) :
+ _hrSorted(hrSorted), _chunk_size(chunk_size), _worker(worker),
+ _marked_regions_added(0), _cur_chunk_idx(0), _cur_chunk_end(0),
+ _invokes(0)
+ {}
+
+ bool doHeapRegion(HeapRegion* r) {
+ // We only include humongous regions in collection
+ // sets when concurrent mark shows that their contained object is
+ // unreachable.
+ _invokes++;
+
+ // Do we have any marking information for this region?
+ if (r->is_marked()) {
+ // We don't include humongous regions in collection
+ // sets because we collect them immediately at the end of a marking
+ // cycle.
+ // We also do not include young regions in collection sets
+ if (!r->isHumongous() && !r->is_young()) {
+ add_region(r);
+ }
+ }
+ return false;
+ }
+ jint marked_regions_added() { return _marked_regions_added; }
+ int invokes() { return _invokes; }
+};
+
+class ParKnownGarbageTask: public AbstractGangTask {
+ CollectionSetChooser* _hrSorted;
+ jint _chunk_size;
+ G1CollectedHeap* _g1;
+public:
+ ParKnownGarbageTask(CollectionSetChooser* hrSorted, jint chunk_size) :
+ AbstractGangTask("ParKnownGarbageTask"),
+ _hrSorted(hrSorted), _chunk_size(chunk_size),
+ _g1(G1CollectedHeap::heap())
+ {}
+
+ void work(int i) {
+ ParKnownGarbageHRClosure parKnownGarbageCl(_hrSorted, _chunk_size, i);
+ // Back to zero for the claim value.
+ _g1->heap_region_par_iterate_chunked(&parKnownGarbageCl, i,
+ HeapRegion::InitialClaimValue);
+ jint regions_added = parKnownGarbageCl.marked_regions_added();
+ _hrSorted->incNumMarkedHeapRegions(regions_added);
+ if (G1PrintParCleanupStats) {
+ gclog_or_tty->print(" Thread %d called %d times, added %d regions to list.\n",
+ i, parKnownGarbageCl.invokes(), regions_added);
+ }
+ }
+};
+
+void
+G1CollectorPolicy_BestRegionsFirst::
+record_concurrent_mark_cleanup_end(size_t freed_bytes,
+ size_t max_live_bytes) {
+ double start;
+ if (G1PrintParCleanupStats) start = os::elapsedTime();
+ record_concurrent_mark_cleanup_end_work1(freed_bytes, max_live_bytes);
+
+ _collectionSetChooser->clearMarkedHeapRegions();
+ double clear_marked_end;
+ if (G1PrintParCleanupStats) {
+ clear_marked_end = os::elapsedTime();
+ gclog_or_tty->print_cr(" clear marked regions + work1: %8.3f ms.",
+ (clear_marked_end - start)*1000.0);
+ }
+ if (ParallelGCThreads > 0) {
+ const size_t OverpartitionFactor = 4;
+ const size_t MinChunkSize = 8;
+ const size_t ChunkSize =
+ MAX2(_g1->n_regions() / (ParallelGCThreads * OverpartitionFactor),
+ MinChunkSize);
+ _collectionSetChooser->prepareForAddMarkedHeapRegionsPar(_g1->n_regions(),
+ ChunkSize);
+ ParKnownGarbageTask parKnownGarbageTask(_collectionSetChooser,
+ (int) ChunkSize);
+ _g1->workers()->run_task(&parKnownGarbageTask);
+
+ assert(_g1->check_heap_region_claim_values(HeapRegion::InitialClaimValue),
+ "sanity check");
+ } else {
+ KnownGarbageClosure knownGarbagecl(_collectionSetChooser);
+ _g1->heap_region_iterate(&knownGarbagecl);
+ }
+ double known_garbage_end;
+ if (G1PrintParCleanupStats) {
+ known_garbage_end = os::elapsedTime();
+ gclog_or_tty->print_cr(" compute known garbage: %8.3f ms.",
+ (known_garbage_end - clear_marked_end)*1000.0);
+ }
+ _collectionSetChooser->sortMarkedHeapRegions();
+ double sort_end;
+ if (G1PrintParCleanupStats) {
+ sort_end = os::elapsedTime();
+ gclog_or_tty->print_cr(" sorting: %8.3f ms.",
+ (sort_end - known_garbage_end)*1000.0);
+ }
+
+ record_concurrent_mark_cleanup_end_work2();
+ double work2_end;
+ if (G1PrintParCleanupStats) {
+ work2_end = os::elapsedTime();
+ gclog_or_tty->print_cr(" work2: %8.3f ms.",
+ (work2_end - sort_end)*1000.0);
+ }
+}
+
+// Add the heap region to the collection set and return the conservative
+// estimate of the number of live bytes.
+void G1CollectorPolicy::
+add_to_collection_set(HeapRegion* hr) {
+ if (G1TraceRegions) {
+ gclog_or_tty->print_cr("added region to cset %d:["PTR_FORMAT", "PTR_FORMAT"], "
+ "top "PTR_FORMAT", young %s",
+ hr->hrs_index(), hr->bottom(), hr->end(),
+ hr->top(), (hr->is_young()) ? "YES" : "NO");
+ }
+
+ if (_g1->mark_in_progress())
+ _g1->concurrent_mark()->registerCSetRegion(hr);
+
+ assert(!hr->in_collection_set(),
+ "should not already be in the CSet");
+ hr->set_in_collection_set(true);
+ hr->set_next_in_collection_set(_collection_set);
+ _collection_set = hr;
+ _collection_set_size++;
+ _collection_set_bytes_used_before += hr->used();
+}
+
+void
+G1CollectorPolicy_BestRegionsFirst::
+choose_collection_set(HeapRegion* pop_region) {
+ double non_young_start_time_sec;
+ start_recording_regions();
+
+ if (pop_region != NULL) {
+ _target_pause_time_ms = (double) G1MaxPauseTimeMS;
+ } else {
+ guarantee(_target_pause_time_ms > -1.0,
+ "_target_pause_time_ms should have been set!");
+ }
+
+ // pop region is either null (and so is CS), or else it *is* the CS.
+ assert(_collection_set == pop_region, "Precondition");
+
+ double base_time_ms = predict_base_elapsed_time_ms(_pending_cards);
+ double predicted_pause_time_ms = base_time_ms;
+
+ double target_time_ms = _target_pause_time_ms;
+ double time_remaining_ms = target_time_ms - base_time_ms;
+
+ // the 10% and 50% values are arbitrary...
+ if (time_remaining_ms < 0.10*target_time_ms) {
+ time_remaining_ms = 0.50 * target_time_ms;
+ _within_target = false;
+ } else {
+ _within_target = true;
+ }
+
+ // We figure out the number of bytes available for future to-space.
+ // For new regions without marking information, we must assume the
+ // worst-case of complete survival. If we have marking information for a
+ // region, we can bound the amount of live data. We can add a number of
+ // such regions, as long as the sum of the live data bounds does not
+ // exceed the available evacuation space.
+ size_t max_live_bytes = _g1->free_regions() * HeapRegion::GrainBytes;
+
+ size_t expansion_bytes =
+ _g1->expansion_regions() * HeapRegion::GrainBytes;
+
+ if (pop_region == NULL) {
+ _collection_set_bytes_used_before = 0;
+ _collection_set_size = 0;
+ }
+
+ // Adjust for expansion and slop.
+ max_live_bytes = max_live_bytes + expansion_bytes;
+
+ assert(pop_region != NULL || _g1->regions_accounted_for(), "Region leakage!");
+
+ HeapRegion* hr;
+ if (in_young_gc_mode()) {
+ double young_start_time_sec = os::elapsedTime();
+
+ if (G1PolicyVerbose > 0) {
+ gclog_or_tty->print_cr("Adding %d young regions to the CSet",
+ _g1->young_list_length());
+ }
+ _young_cset_length = 0;
+ _last_young_gc_full = full_young_gcs() ? true : false;
+ if (_last_young_gc_full)
+ ++_full_young_pause_num;
+ else
+ ++_partial_young_pause_num;
+ hr = _g1->pop_region_from_young_list();
+ while (hr != NULL) {
+
+ assert( hr->young_index_in_cset() == -1, "invariant" );
+ assert( hr->age_in_surv_rate_group() != -1, "invariant" );
+ hr->set_young_index_in_cset((int) _young_cset_length);
+
+ ++_young_cset_length;
+ double predicted_time_ms = predict_region_elapsed_time_ms(hr, true);
+ time_remaining_ms -= predicted_time_ms;
+ predicted_pause_time_ms += predicted_time_ms;
+ if (hr == pop_region) {
+ // The popular region was young. Skip over it.
+ assert(hr->in_collection_set(), "It's the pop region.");
+ } else {
+ assert(!hr->in_collection_set(), "It's not the pop region.");
+ add_to_collection_set(hr);
+ record_cset_region(hr, true);
+ }
+ max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes);
+ if (G1PolicyVerbose > 0) {
+ gclog_or_tty->print_cr(" Added [" PTR_FORMAT ", " PTR_FORMAT") to CS.",
+ hr->bottom(), hr->end());
+ gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)",
+ max_live_bytes/K);
+ }
+ hr = _g1->pop_region_from_young_list();
+ }
+
+ record_scan_only_regions(_g1->young_list_scan_only_length());
+
+ double young_end_time_sec = os::elapsedTime();
+ _recorded_young_cset_choice_time_ms =
+ (young_end_time_sec - young_start_time_sec) * 1000.0;
+
+ non_young_start_time_sec = os::elapsedTime();
+
+ if (_young_cset_length > 0 && _last_young_gc_full) {
+ // don't bother adding more regions...
+ goto choose_collection_set_end;
+ }
+ } else if (pop_region != NULL) {
+ // We're not in young mode, and we chose a popular region; don't choose
+ // any more.
+ return;
+ }
+
+ if (!in_young_gc_mode() || !full_young_gcs()) {
+ bool should_continue = true;
+ NumberSeq seq;
+ double avg_prediction = 100000000000000000.0; // something very large
+ do {
+ hr = _collectionSetChooser->getNextMarkedRegion(time_remaining_ms,
+ avg_prediction);
+ if (hr != NULL && !hr->popular()) {
+ double predicted_time_ms = predict_region_elapsed_time_ms(hr, false);
+ time_remaining_ms -= predicted_time_ms;
+ predicted_pause_time_ms += predicted_time_ms;
+ add_to_collection_set(hr);
+ record_cset_region(hr, false);
+ max_live_bytes -= MIN2(hr->max_live_bytes(), max_live_bytes);
+ if (G1PolicyVerbose > 0) {
+ gclog_or_tty->print_cr(" (" SIZE_FORMAT " KB left in heap.)",
+ max_live_bytes/K);
+ }
+ seq.add(predicted_time_ms);
+ avg_prediction = seq.avg() + seq.sd();
+ }
+ should_continue =
+ ( hr != NULL) &&
+ ( (adaptive_young_list_length()) ? time_remaining_ms > 0.0
+ : _collection_set_size < _young_list_fixed_length );
+ } while (should_continue);
+
+ if (!adaptive_young_list_length() &&
+ _collection_set_size < _young_list_fixed_length)
+ _should_revert_to_full_young_gcs = true;
+ }
+
+choose_collection_set_end:
+ count_CS_bytes_used();
+
+ end_recording_regions();
+
+ double non_young_end_time_sec = os::elapsedTime();
+ _recorded_non_young_cset_choice_time_ms =
+ (non_young_end_time_sec - non_young_start_time_sec) * 1000.0;
+}
+
+void G1CollectorPolicy_BestRegionsFirst::record_full_collection_end() {
+ G1CollectorPolicy::record_full_collection_end();
+ _collectionSetChooser->updateAfterFullCollection();
+}
+
+void G1CollectorPolicy_BestRegionsFirst::
+expand_if_possible(size_t numRegions) {
+ size_t expansion_bytes = numRegions * HeapRegion::GrainBytes;
+ _g1->expand(expansion_bytes);
+}
+
+void G1CollectorPolicy_BestRegionsFirst::
+record_collection_pause_end(bool popular, bool abandoned) {
+ G1CollectorPolicy::record_collection_pause_end(popular, abandoned);
+ assert(assertMarkedBytesDataOK(), "Marked regions not OK at pause end.");
+}
+
+// Local Variables: ***
+// c-indentation-style: gnu ***
+// End: ***
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp
new file mode 100644
index 00000000000..110f81fef0e
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp
@@ -0,0 +1,1199 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// A G1CollectorPolicy makes policy decisions that determine the
+// characteristics of the collector. Examples include:
+// * choice of collection set.
+// * when to collect.
+
+class HeapRegion;
+class CollectionSetChooser;
+
+// Yes, this is a bit unpleasant... but it saves replicating the same thing
+// over and over again and introducing subtle problems through small typos and
+// cutting and pasting mistakes. The macros below introduces a number
+// sequnce into the following two classes and the methods that access it.
+
+#define define_num_seq(name) \
+private: \
+ NumberSeq _all_##name##_times_ms; \
+public: \
+ void record_##name##_time_ms(double ms) { \
+ _all_##name##_times_ms.add(ms); \
+ } \
+ NumberSeq* get_##name##_seq() { \
+ return &_all_##name##_times_ms; \
+ }
+
+class MainBodySummary;
+class PopPreambleSummary;
+
+class PauseSummary {
+ define_num_seq(total)
+ define_num_seq(other)
+
+public:
+ virtual MainBodySummary* main_body_summary() { return NULL; }
+ virtual PopPreambleSummary* pop_preamble_summary() { return NULL; }
+};
+
+class MainBodySummary {
+ define_num_seq(satb_drain) // optional
+ define_num_seq(parallel) // parallel only
+ define_num_seq(ext_root_scan)
+ define_num_seq(mark_stack_scan)
+ define_num_seq(scan_only)
+ define_num_seq(update_rs)
+ define_num_seq(scan_rs)
+ define_num_seq(scan_new_refs) // Only for temp use; added to
+ // in parallel case.
+ define_num_seq(obj_copy)
+ define_num_seq(termination) // parallel only
+ define_num_seq(parallel_other) // parallel only
+ define_num_seq(mark_closure)
+ define_num_seq(clear_ct) // parallel only
+};
+
+class PopPreambleSummary {
+ define_num_seq(pop_preamble)
+ define_num_seq(pop_update_rs)
+ define_num_seq(pop_scan_rs)
+ define_num_seq(pop_closure_app)
+ define_num_seq(pop_evacuation)
+ define_num_seq(pop_other)
+};
+
+class NonPopSummary: public PauseSummary,
+ public MainBodySummary {
+public:
+ virtual MainBodySummary* main_body_summary() { return this; }
+};
+
+class PopSummary: public PauseSummary,
+ public MainBodySummary,
+ public PopPreambleSummary {
+public:
+ virtual MainBodySummary* main_body_summary() { return this; }
+ virtual PopPreambleSummary* pop_preamble_summary() { return this; }
+};
+
+class NonPopAbandonedSummary: public PauseSummary {
+};
+
+class PopAbandonedSummary: public PauseSummary,
+ public PopPreambleSummary {
+public:
+ virtual PopPreambleSummary* pop_preamble_summary() { return this; }
+};
+
+class G1CollectorPolicy: public CollectorPolicy {
+protected:
+ // The number of pauses during the execution.
+ long _n_pauses;
+
+ // either equal to the number of parallel threads, if ParallelGCThreads
+ // has been set, or 1 otherwise
+ int _parallel_gc_threads;
+
+ enum SomePrivateConstants {
+ NumPrevPausesForHeuristics = 10,
+ NumPrevGCsForHeuristics = 10,
+ NumAPIs = HeapRegion::MaxAge
+ };
+
+ G1MMUTracker* _mmu_tracker;
+
+ void initialize_flags();
+
+ void initialize_all() {
+ initialize_flags();
+ initialize_size_info();
+ initialize_perm_generation(PermGen::MarkSweepCompact);
+ }
+
+ virtual size_t default_init_heap_size() {
+ // Pick some reasonable default.
+ return 8*M;
+ }
+
+
+ double _cur_collection_start_sec;
+ size_t _cur_collection_pause_used_at_start_bytes;
+ size_t _cur_collection_pause_used_regions_at_start;
+ size_t _prev_collection_pause_used_at_end_bytes;
+ double _cur_collection_par_time_ms;
+ double _cur_satb_drain_time_ms;
+ double _cur_clear_ct_time_ms;
+ bool _satb_drain_time_set;
+ double _cur_popular_preamble_start_ms;
+ double _cur_popular_preamble_time_ms;
+ double _cur_popular_compute_rc_time_ms;
+ double _cur_popular_evac_time_ms;
+
+ double _cur_CH_strong_roots_end_sec;
+ double _cur_CH_strong_roots_dur_ms;
+ double _cur_G1_strong_roots_end_sec;
+ double _cur_G1_strong_roots_dur_ms;
+
+ // Statistics for recent GC pauses. See below for how indexed.
+ TruncatedSeq* _recent_CH_strong_roots_times_ms;
+ TruncatedSeq* _recent_G1_strong_roots_times_ms;
+ TruncatedSeq* _recent_evac_times_ms;
+ // These exclude marking times.
+ TruncatedSeq* _recent_pause_times_ms;
+ TruncatedSeq* _recent_gc_times_ms;
+
+ TruncatedSeq* _recent_CS_bytes_used_before;
+ TruncatedSeq* _recent_CS_bytes_surviving;
+
+ TruncatedSeq* _recent_rs_sizes;
+
+ TruncatedSeq* _concurrent_mark_init_times_ms;
+ TruncatedSeq* _concurrent_mark_remark_times_ms;
+ TruncatedSeq* _concurrent_mark_cleanup_times_ms;
+
+ NonPopSummary* _non_pop_summary;
+ PopSummary* _pop_summary;
+ NonPopAbandonedSummary* _non_pop_abandoned_summary;
+ PopAbandonedSummary* _pop_abandoned_summary;
+
+ NumberSeq* _all_pause_times_ms;
+ NumberSeq* _all_full_gc_times_ms;
+ double _stop_world_start;
+ NumberSeq* _all_stop_world_times_ms;
+ NumberSeq* _all_yield_times_ms;
+
+ size_t _region_num_young;
+ size_t _region_num_tenured;
+ size_t _prev_region_num_young;
+ size_t _prev_region_num_tenured;
+
+ NumberSeq* _all_mod_union_times_ms;
+
+ int _aux_num;
+ NumberSeq* _all_aux_times_ms;
+ double* _cur_aux_start_times_ms;
+ double* _cur_aux_times_ms;
+ bool* _cur_aux_times_set;
+
+ double* _par_last_ext_root_scan_times_ms;
+ double* _par_last_mark_stack_scan_times_ms;
+ double* _par_last_scan_only_times_ms;
+ double* _par_last_scan_only_regions_scanned;
+ double* _par_last_update_rs_start_times_ms;
+ double* _par_last_update_rs_times_ms;
+ double* _par_last_update_rs_processed_buffers;
+ double* _par_last_scan_rs_start_times_ms;
+ double* _par_last_scan_rs_times_ms;
+ double* _par_last_scan_new_refs_times_ms;
+ double* _par_last_obj_copy_times_ms;
+ double* _par_last_termination_times_ms;
+
+ // there are two pases during popular pauses, so we need to store
+ // somewhere the results of the first pass
+ double* _pop_par_last_update_rs_start_times_ms;
+ double* _pop_par_last_update_rs_times_ms;
+ double* _pop_par_last_update_rs_processed_buffers;
+ double* _pop_par_last_scan_rs_start_times_ms;
+ double* _pop_par_last_scan_rs_times_ms;
+ double* _pop_par_last_closure_app_times_ms;
+
+ double _pop_compute_rc_start;
+ double _pop_evac_start;
+
+ // indicates that we are in young GC mode
+ bool _in_young_gc_mode;
+
+ // indicates whether we are in full young or partially young GC mode
+ bool _full_young_gcs;
+
+ // if true, then it tries to dynamically adjust the length of the
+ // young list
+ bool _adaptive_young_list_length;
+ size_t _young_list_min_length;
+ size_t _young_list_target_length;
+ size_t _young_list_so_prefix_length;
+ size_t _young_list_fixed_length;
+
+ size_t _young_cset_length;
+ bool _last_young_gc_full;
+
+ double _target_pause_time_ms;
+
+ unsigned _full_young_pause_num;
+ unsigned _partial_young_pause_num;
+
+ bool _during_marking;
+ bool _in_marking_window;
+ bool _in_marking_window_im;
+
+ SurvRateGroup* _short_lived_surv_rate_group;
+ SurvRateGroup* _survivor_surv_rate_group;
+ // add here any more surv rate groups
+
+ bool during_marking() {
+ return _during_marking;
+ }
+
+ //
+
+private:
+ enum PredictionConstants {
+ TruncatedSeqLength = 10
+ };
+
+ TruncatedSeq* _alloc_rate_ms_seq;
+ double _prev_collection_pause_end_ms;
+
+ TruncatedSeq* _pending_card_diff_seq;
+ TruncatedSeq* _rs_length_diff_seq;
+ TruncatedSeq* _cost_per_card_ms_seq;
+ TruncatedSeq* _cost_per_scan_only_region_ms_seq;
+ TruncatedSeq* _fully_young_cards_per_entry_ratio_seq;
+ TruncatedSeq* _partially_young_cards_per_entry_ratio_seq;
+ TruncatedSeq* _cost_per_entry_ms_seq;
+ TruncatedSeq* _partially_young_cost_per_entry_ms_seq;
+ TruncatedSeq* _cost_per_byte_ms_seq;
+ TruncatedSeq* _constant_other_time_ms_seq;
+ TruncatedSeq* _young_other_cost_per_region_ms_seq;
+ TruncatedSeq* _non_young_other_cost_per_region_ms_seq;
+
+ TruncatedSeq* _pending_cards_seq;
+ TruncatedSeq* _scanned_cards_seq;
+ TruncatedSeq* _rs_lengths_seq;
+
+ TruncatedSeq* _cost_per_byte_ms_during_cm_seq;
+ TruncatedSeq* _cost_per_scan_only_region_ms_during_cm_seq;
+
+ TruncatedSeq* _young_gc_eff_seq;
+
+ TruncatedSeq* _max_conc_overhead_seq;
+
+ size_t _recorded_young_regions;
+ size_t _recorded_scan_only_regions;
+ size_t _recorded_non_young_regions;
+ size_t _recorded_region_num;
+
+ size_t _free_regions_at_end_of_collection;
+ size_t _scan_only_regions_at_end_of_collection;
+
+ size_t _recorded_rs_lengths;
+ size_t _max_rs_lengths;
+
+ size_t _recorded_marked_bytes;
+ size_t _recorded_young_bytes;
+
+ size_t _predicted_pending_cards;
+ size_t _predicted_cards_scanned;
+ size_t _predicted_rs_lengths;
+ size_t _predicted_bytes_to_copy;
+
+ double _predicted_survival_ratio;
+ double _predicted_rs_update_time_ms;
+ double _predicted_rs_scan_time_ms;
+ double _predicted_scan_only_scan_time_ms;
+ double _predicted_object_copy_time_ms;
+ double _predicted_constant_other_time_ms;
+ double _predicted_young_other_time_ms;
+ double _predicted_non_young_other_time_ms;
+ double _predicted_pause_time_ms;
+
+ double _vtime_diff_ms;
+
+ double _recorded_young_free_cset_time_ms;
+ double _recorded_non_young_free_cset_time_ms;
+
+ double _sigma;
+ double _expensive_region_limit_ms;
+
+ size_t _rs_lengths_prediction;
+
+ size_t _known_garbage_bytes;
+ double _known_garbage_ratio;
+
+ double sigma() {
+ return _sigma;
+ }
+
+ // A function that prevents us putting too much stock in small sample
+ // sets. Returns a number between 2.0 and 1.0, depending on the number
+ // of samples. 5 or more samples yields one; fewer scales linearly from
+ // 2.0 at 1 sample to 1.0 at 5.
+ double confidence_factor(int samples) {
+ if (samples > 4) return 1.0;
+ else return 1.0 + sigma() * ((double)(5 - samples))/2.0;
+ }
+
+ double get_new_neg_prediction(TruncatedSeq* seq) {
+ return seq->davg() - sigma() * seq->dsd();
+ }
+
+#ifndef PRODUCT
+ bool verify_young_ages(HeapRegion* head, SurvRateGroup *surv_rate_group);
+#endif // PRODUCT
+
+protected:
+ double _pause_time_target_ms;
+ double _recorded_young_cset_choice_time_ms;
+ double _recorded_non_young_cset_choice_time_ms;
+ bool _within_target;
+ size_t _pending_cards;
+ size_t _max_pending_cards;
+
+public:
+
+ void set_region_short_lived(HeapRegion* hr) {
+ hr->install_surv_rate_group(_short_lived_surv_rate_group);
+ }
+
+ void set_region_survivors(HeapRegion* hr) {
+ hr->install_surv_rate_group(_survivor_surv_rate_group);
+ }
+
+#ifndef PRODUCT
+ bool verify_young_ages();
+#endif // PRODUCT
+
+ void tag_scan_only(size_t short_lived_scan_only_length);
+
+ double get_new_prediction(TruncatedSeq* seq) {
+ return MAX2(seq->davg() + sigma() * seq->dsd(),
+ seq->davg() * confidence_factor(seq->num()));
+ }
+
+ size_t young_cset_length() {
+ return _young_cset_length;
+ }
+
+ void record_max_rs_lengths(size_t rs_lengths) {
+ _max_rs_lengths = rs_lengths;
+ }
+
+ size_t predict_pending_card_diff() {
+ double prediction = get_new_neg_prediction(_pending_card_diff_seq);
+ if (prediction < 0.00001)
+ return 0;
+ else
+ return (size_t) prediction;
+ }
+
+ size_t predict_pending_cards() {
+ size_t max_pending_card_num = _g1->max_pending_card_num();
+ size_t diff = predict_pending_card_diff();
+ size_t prediction;
+ if (diff > max_pending_card_num)
+ prediction = max_pending_card_num;
+ else
+ prediction = max_pending_card_num - diff;
+
+ return prediction;
+ }
+
+ size_t predict_rs_length_diff() {
+ return (size_t) get_new_prediction(_rs_length_diff_seq);
+ }
+
+ double predict_alloc_rate_ms() {
+ return get_new_prediction(_alloc_rate_ms_seq);
+ }
+
+ double predict_cost_per_card_ms() {
+ return get_new_prediction(_cost_per_card_ms_seq);
+ }
+
+ double predict_rs_update_time_ms(size_t pending_cards) {
+ return (double) pending_cards * predict_cost_per_card_ms();
+ }
+
+ double predict_fully_young_cards_per_entry_ratio() {
+ return get_new_prediction(_fully_young_cards_per_entry_ratio_seq);
+ }
+
+ double predict_partially_young_cards_per_entry_ratio() {
+ if (_partially_young_cards_per_entry_ratio_seq->num() < 2)
+ return predict_fully_young_cards_per_entry_ratio();
+ else
+ return get_new_prediction(_partially_young_cards_per_entry_ratio_seq);
+ }
+
+ size_t predict_young_card_num(size_t rs_length) {
+ return (size_t) ((double) rs_length *
+ predict_fully_young_cards_per_entry_ratio());
+ }
+
+ size_t predict_non_young_card_num(size_t rs_length) {
+ return (size_t) ((double) rs_length *
+ predict_partially_young_cards_per_entry_ratio());
+ }
+
+ double predict_rs_scan_time_ms(size_t card_num) {
+ if (full_young_gcs())
+ return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq);
+ else
+ return predict_partially_young_rs_scan_time_ms(card_num);
+ }
+
+ double predict_partially_young_rs_scan_time_ms(size_t card_num) {
+ if (_partially_young_cost_per_entry_ms_seq->num() < 3)
+ return (double) card_num * get_new_prediction(_cost_per_entry_ms_seq);
+ else
+ return (double) card_num *
+ get_new_prediction(_partially_young_cost_per_entry_ms_seq);
+ }
+
+ double predict_scan_only_time_ms_during_cm(size_t scan_only_region_num) {
+ if (_cost_per_scan_only_region_ms_during_cm_seq->num() < 3)
+ return 1.5 * (double) scan_only_region_num *
+ get_new_prediction(_cost_per_scan_only_region_ms_seq);
+ else
+ return (double) scan_only_region_num *
+ get_new_prediction(_cost_per_scan_only_region_ms_during_cm_seq);
+ }
+
+ double predict_scan_only_time_ms(size_t scan_only_region_num) {
+ if (_in_marking_window_im)
+ return predict_scan_only_time_ms_during_cm(scan_only_region_num);
+ else
+ return (double) scan_only_region_num *
+ get_new_prediction(_cost_per_scan_only_region_ms_seq);
+ }
+
+ double predict_object_copy_time_ms_during_cm(size_t bytes_to_copy) {
+ if (_cost_per_byte_ms_during_cm_seq->num() < 3)
+ return 1.1 * (double) bytes_to_copy *
+ get_new_prediction(_cost_per_byte_ms_seq);
+ else
+ return (double) bytes_to_copy *
+ get_new_prediction(_cost_per_byte_ms_during_cm_seq);
+ }
+
+ double predict_object_copy_time_ms(size_t bytes_to_copy) {
+ if (_in_marking_window && !_in_marking_window_im)
+ return predict_object_copy_time_ms_during_cm(bytes_to_copy);
+ else
+ return (double) bytes_to_copy *
+ get_new_prediction(_cost_per_byte_ms_seq);
+ }
+
+ double predict_constant_other_time_ms() {
+ return get_new_prediction(_constant_other_time_ms_seq);
+ }
+
+ double predict_young_other_time_ms(size_t young_num) {
+ return
+ (double) young_num *
+ get_new_prediction(_young_other_cost_per_region_ms_seq);
+ }
+
+ double predict_non_young_other_time_ms(size_t non_young_num) {
+ return
+ (double) non_young_num *
+ get_new_prediction(_non_young_other_cost_per_region_ms_seq);
+ }
+
+ void check_if_region_is_too_expensive(double predicted_time_ms);
+
+ double predict_young_collection_elapsed_time_ms(size_t adjustment);
+ double predict_base_elapsed_time_ms(size_t pending_cards);
+ double predict_base_elapsed_time_ms(size_t pending_cards,
+ size_t scanned_cards);
+ size_t predict_bytes_to_copy(HeapRegion* hr);
+ double predict_region_elapsed_time_ms(HeapRegion* hr, bool young);
+
+ // for use by: calculate_optimal_so_length(length)
+ void predict_gc_eff(size_t young_region_num,
+ size_t so_length,
+ double base_time_ms,
+ double *gc_eff,
+ double *pause_time_ms);
+
+ // for use by: calculate_young_list_target_config(rs_length)
+ bool predict_gc_eff(size_t young_region_num,
+ size_t so_length,
+ double base_time_with_so_ms,
+ size_t init_free_regions,
+ double target_pause_time_ms,
+ double* gc_eff);
+
+ void start_recording_regions();
+ void record_cset_region(HeapRegion* hr, bool young);
+ void record_scan_only_regions(size_t scan_only_length);
+ void end_recording_regions();
+
+ void record_vtime_diff_ms(double vtime_diff_ms) {
+ _vtime_diff_ms = vtime_diff_ms;
+ }
+
+ void record_young_free_cset_time_ms(double time_ms) {
+ _recorded_young_free_cset_time_ms = time_ms;
+ }
+
+ void record_non_young_free_cset_time_ms(double time_ms) {
+ _recorded_non_young_free_cset_time_ms = time_ms;
+ }
+
+ double predict_young_gc_eff() {
+ return get_new_neg_prediction(_young_gc_eff_seq);
+ }
+
+ //
+
+public:
+ void cset_regions_freed() {
+ bool propagate = _last_young_gc_full && !_in_marking_window;
+ _short_lived_surv_rate_group->all_surviving_words_recorded(propagate);
+ _survivor_surv_rate_group->all_surviving_words_recorded(propagate);
+ // also call it on any more surv rate groups
+ }
+
+ void set_known_garbage_bytes(size_t known_garbage_bytes) {
+ _known_garbage_bytes = known_garbage_bytes;
+ size_t heap_bytes = _g1->capacity();
+ _known_garbage_ratio = (double) _known_garbage_bytes / (double) heap_bytes;
+ }
+
+ void decrease_known_garbage_bytes(size_t known_garbage_bytes) {
+ guarantee( _known_garbage_bytes >= known_garbage_bytes, "invariant" );
+
+ _known_garbage_bytes -= known_garbage_bytes;
+ size_t heap_bytes = _g1->capacity();
+ _known_garbage_ratio = (double) _known_garbage_bytes / (double) heap_bytes;
+ }
+
+ G1MMUTracker* mmu_tracker() {
+ return _mmu_tracker;
+ }
+
+ double predict_init_time_ms() {
+ return get_new_prediction(_concurrent_mark_init_times_ms);
+ }
+
+ double predict_remark_time_ms() {
+ return get_new_prediction(_concurrent_mark_remark_times_ms);
+ }
+
+ double predict_cleanup_time_ms() {
+ return get_new_prediction(_concurrent_mark_cleanup_times_ms);
+ }
+
+ // Returns an estimate of the survival rate of the region at yg-age
+ // "yg_age".
+ double predict_yg_surv_rate(int age) {
+ TruncatedSeq* seq = _short_lived_surv_rate_group->get_seq(age);
+ if (seq->num() == 0)
+ gclog_or_tty->print("BARF! age is %d", age);
+ guarantee( seq->num() > 0, "invariant" );
+ double pred = get_new_prediction(seq);
+ if (pred > 1.0)
+ pred = 1.0;
+ return pred;
+ }
+
+ double accum_yg_surv_rate_pred(int age) {
+ return _short_lived_surv_rate_group->accum_surv_rate_pred(age);
+ }
+
+protected:
+ void print_stats (int level, const char* str, double value);
+ void print_stats (int level, const char* str, int value);
+ void print_par_stats (int level, const char* str, double* data) {
+ print_par_stats(level, str, data, true);
+ }
+ void print_par_stats (int level, const char* str, double* data, bool summary);
+ void print_par_buffers (int level, const char* str, double* data, bool summary);
+
+ void check_other_times(int level,
+ NumberSeq* other_times_ms,
+ NumberSeq* calc_other_times_ms) const;
+
+ void print_summary (PauseSummary* stats) const;
+ void print_abandoned_summary(PauseSummary* non_pop_summary,
+ PauseSummary* pop_summary) const;
+
+ void print_summary (int level, const char* str, NumberSeq* seq) const;
+ void print_summary_sd (int level, const char* str, NumberSeq* seq) const;
+
+ double avg_value (double* data);
+ double max_value (double* data);
+ double sum_of_values (double* data);
+ double max_sum (double* data1, double* data2);
+
+ int _last_satb_drain_processed_buffers;
+ int _last_update_rs_processed_buffers;
+ double _last_pause_time_ms;
+
+ size_t _bytes_in_to_space_before_gc;
+ size_t _bytes_in_to_space_after_gc;
+ size_t bytes_in_to_space_during_gc() {
+ return
+ _bytes_in_to_space_after_gc - _bytes_in_to_space_before_gc;
+ }
+ size_t _bytes_in_collection_set_before_gc;
+ // Used to count used bytes in CS.
+ friend class CountCSClosure;
+
+ // Statistics kept per GC stoppage, pause or full.
+ TruncatedSeq* _recent_prev_end_times_for_all_gcs_sec;
+
+ // We track markings.
+ int _num_markings;
+ double _mark_thread_startup_sec; // Time at startup of marking thread
+
+ // Add a new GC of the given duration and end time to the record.
+ void update_recent_gc_times(double end_time_sec, double elapsed_ms);
+
+ // The head of the list (via "next_in_collection_set()") representing the
+ // current collection set.
+ HeapRegion* _collection_set;
+ size_t _collection_set_size;
+ size_t _collection_set_bytes_used_before;
+
+ // Info about marking.
+ int _n_marks; // Sticky at 2, so we know when we've done at least 2.
+
+ // The number of collection pauses at the end of the last mark.
+ size_t _n_pauses_at_mark_end;
+
+ // ==== This section is for stats related to starting Conc Refinement on time.
+ size_t _conc_refine_enabled;
+ size_t _conc_refine_zero_traversals;
+ size_t _conc_refine_max_traversals;
+ // In # of heap regions.
+ size_t _conc_refine_current_delta;
+
+ // At the beginning of a collection pause, update the variables above,
+ // especially the "delta".
+ void update_conc_refine_data();
+ // ====
+
+ // Stash a pointer to the g1 heap.
+ G1CollectedHeap* _g1;
+
+ // The average time in ms per collection pause, averaged over recent pauses.
+ double recent_avg_time_for_pauses_ms();
+
+ // The average time in ms for processing CollectedHeap strong roots, per
+ // collection pause, averaged over recent pauses.
+ double recent_avg_time_for_CH_strong_ms();
+
+ // The average time in ms for processing the G1 remembered set, per
+ // pause, averaged over recent pauses.
+ double recent_avg_time_for_G1_strong_ms();
+
+ // The average time in ms for "evacuating followers", per pause, averaged
+ // over recent pauses.
+ double recent_avg_time_for_evac_ms();
+
+ // The number of "recent" GCs recorded in the number sequences
+ int number_of_recent_gcs();
+
+ // The average survival ratio, computed by the total number of bytes
+ // suriviving / total number of bytes before collection over the last
+ // several recent pauses.
+ double recent_avg_survival_fraction();
+ // The survival fraction of the most recent pause; if there have been no
+ // pauses, returns 1.0.
+ double last_survival_fraction();
+
+ // Returns a "conservative" estimate of the recent survival rate, i.e.,
+ // one that may be higher than "recent_avg_survival_fraction".
+ // This is conservative in several ways:
+ // If there have been few pauses, it will assume a potential high
+ // variance, and err on the side of caution.
+ // It puts a lower bound (currently 0.1) on the value it will return.
+ // To try to detect phase changes, if the most recent pause ("latest") has a
+ // higher-than average ("avg") survival rate, it returns that rate.
+ // "work" version is a utility function; young is restricted to young regions.
+ double conservative_avg_survival_fraction_work(double avg,
+ double latest);
+
+ // The arguments are the two sequences that keep track of the number of bytes
+ // surviving and the total number of bytes before collection, resp.,
+ // over the last evereal recent pauses
+ // Returns the survival rate for the category in the most recent pause.
+ // If there have been no pauses, returns 1.0.
+ double last_survival_fraction_work(TruncatedSeq* surviving,
+ TruncatedSeq* before);
+
+ // The arguments are the two sequences that keep track of the number of bytes
+ // surviving and the total number of bytes before collection, resp.,
+ // over the last several recent pauses
+ // Returns the average survival ration over the last several recent pauses
+ // If there have been no pauses, return 1.0
+ double recent_avg_survival_fraction_work(TruncatedSeq* surviving,
+ TruncatedSeq* before);
+
+ double conservative_avg_survival_fraction() {
+ double avg = recent_avg_survival_fraction();
+ double latest = last_survival_fraction();
+ return conservative_avg_survival_fraction_work(avg, latest);
+ }
+
+ // The ratio of gc time to elapsed time, computed over recent pauses.
+ double _recent_avg_pause_time_ratio;
+
+ double recent_avg_pause_time_ratio() {
+ return _recent_avg_pause_time_ratio;
+ }
+
+ // Number of pauses between concurrent marking.
+ size_t _pauses_btwn_concurrent_mark;
+
+ size_t _n_marks_since_last_pause;
+
+ // True iff CM has been initiated.
+ bool _conc_mark_initiated;
+
+ // True iff CM should be initiated
+ bool _should_initiate_conc_mark;
+ bool _should_revert_to_full_young_gcs;
+ bool _last_full_young_gc;
+
+ // This set of variables tracks the collector efficiency, in order to
+ // determine whether we should initiate a new marking.
+ double _cur_mark_stop_world_time_ms;
+ double _mark_init_start_sec;
+ double _mark_remark_start_sec;
+ double _mark_cleanup_start_sec;
+ double _mark_closure_time_ms;
+
+ void calculate_young_list_min_length();
+ void calculate_young_list_target_config();
+ void calculate_young_list_target_config(size_t rs_lengths);
+ size_t calculate_optimal_so_length(size_t young_list_length);
+
+public:
+
+ G1CollectorPolicy();
+
+ virtual G1CollectorPolicy* as_g1_policy() { return this; }
+
+ virtual CollectorPolicy::Name kind() {
+ return CollectorPolicy::G1CollectorPolicyKind;
+ }
+
+ void check_prediction_validity();
+
+ size_t bytes_in_collection_set() {
+ return _bytes_in_collection_set_before_gc;
+ }
+
+ size_t bytes_in_to_space() {
+ return bytes_in_to_space_during_gc();
+ }
+
+ unsigned calc_gc_alloc_time_stamp() {
+ return _all_pause_times_ms->num() + 1;
+ }
+
+protected:
+
+ // Count the number of bytes used in the CS.
+ void count_CS_bytes_used();
+
+ // Together these do the base cleanup-recording work. Subclasses might
+ // want to put something between them.
+ void record_concurrent_mark_cleanup_end_work1(size_t freed_bytes,
+ size_t max_live_bytes);
+ void record_concurrent_mark_cleanup_end_work2();
+
+public:
+
+ virtual void init();
+
+ virtual HeapWord* mem_allocate_work(size_t size,
+ bool is_tlab,
+ bool* gc_overhead_limit_was_exceeded);
+
+ // This method controls how a collector handles one or more
+ // of its generations being fully allocated.
+ virtual HeapWord* satisfy_failed_allocation(size_t size,
+ bool is_tlab);
+
+ BarrierSet::Name barrier_set_name() { return BarrierSet::G1SATBCTLogging; }
+
+ GenRemSet::Name rem_set_name() { return GenRemSet::CardTable; }
+
+ // The number of collection pauses so far.
+ long n_pauses() const { return _n_pauses; }
+
+ // Update the heuristic info to record a collection pause of the given
+ // start time, where the given number of bytes were used at the start.
+ // This may involve changing the desired size of a collection set.
+
+ virtual void record_stop_world_start();
+
+ virtual void record_collection_pause_start(double start_time_sec,
+ size_t start_used);
+
+ virtual void record_popular_pause_preamble_start();
+ virtual void record_popular_pause_preamble_end();
+
+ // Must currently be called while the world is stopped.
+ virtual void record_concurrent_mark_init_start();
+ virtual void record_concurrent_mark_init_end();
+ void record_concurrent_mark_init_end_pre(double
+ mark_init_elapsed_time_ms);
+
+ void record_mark_closure_time(double mark_closure_time_ms);
+
+ virtual void record_concurrent_mark_remark_start();
+ virtual void record_concurrent_mark_remark_end();
+
+ virtual void record_concurrent_mark_cleanup_start();
+ virtual void record_concurrent_mark_cleanup_end(size_t freed_bytes,
+ size_t max_live_bytes);
+ virtual void record_concurrent_mark_cleanup_completed();
+
+ virtual void record_concurrent_pause();
+ virtual void record_concurrent_pause_end();
+
+ virtual void record_collection_pause_end_CH_strong_roots();
+ virtual void record_collection_pause_end_G1_strong_roots();
+
+ virtual void record_collection_pause_end(bool popular, bool abandoned);
+
+ // Record the fact that a full collection occurred.
+ virtual void record_full_collection_start();
+ virtual void record_full_collection_end();
+
+ void record_ext_root_scan_time(int worker_i, double ms) {
+ _par_last_ext_root_scan_times_ms[worker_i] = ms;
+ }
+
+ void record_mark_stack_scan_time(int worker_i, double ms) {
+ _par_last_mark_stack_scan_times_ms[worker_i] = ms;
+ }
+
+ void record_scan_only_time(int worker_i, double ms, int n) {
+ _par_last_scan_only_times_ms[worker_i] = ms;
+ _par_last_scan_only_regions_scanned[worker_i] = (double) n;
+ }
+
+ void record_satb_drain_time(double ms) {
+ _cur_satb_drain_time_ms = ms;
+ _satb_drain_time_set = true;
+ }
+
+ void record_satb_drain_processed_buffers (int processed_buffers) {
+ _last_satb_drain_processed_buffers = processed_buffers;
+ }
+
+ void record_mod_union_time(double ms) {
+ _all_mod_union_times_ms->add(ms);
+ }
+
+ void record_update_rs_start_time(int thread, double ms) {
+ _par_last_update_rs_start_times_ms[thread] = ms;
+ }
+
+ void record_update_rs_time(int thread, double ms) {
+ _par_last_update_rs_times_ms[thread] = ms;
+ }
+
+ void record_update_rs_processed_buffers (int thread,
+ double processed_buffers) {
+ _par_last_update_rs_processed_buffers[thread] = processed_buffers;
+ }
+
+ void record_scan_rs_start_time(int thread, double ms) {
+ _par_last_scan_rs_start_times_ms[thread] = ms;
+ }
+
+ void record_scan_rs_time(int thread, double ms) {
+ _par_last_scan_rs_times_ms[thread] = ms;
+ }
+
+ void record_scan_new_refs_time(int thread, double ms) {
+ _par_last_scan_new_refs_times_ms[thread] = ms;
+ }
+
+ double get_scan_new_refs_time(int thread) {
+ return _par_last_scan_new_refs_times_ms[thread];
+ }
+
+ void reset_obj_copy_time(int thread) {
+ _par_last_obj_copy_times_ms[thread] = 0.0;
+ }
+
+ void reset_obj_copy_time() {
+ reset_obj_copy_time(0);
+ }
+
+ void record_obj_copy_time(int thread, double ms) {
+ _par_last_obj_copy_times_ms[thread] += ms;
+ }
+
+ void record_obj_copy_time(double ms) {
+ record_obj_copy_time(0, ms);
+ }
+
+ void record_termination_time(int thread, double ms) {
+ _par_last_termination_times_ms[thread] = ms;
+ }
+
+ void record_termination_time(double ms) {
+ record_termination_time(0, ms);
+ }
+
+ void record_pause_time(double ms) {
+ _last_pause_time_ms = ms;
+ }
+
+ void record_clear_ct_time(double ms) {
+ _cur_clear_ct_time_ms = ms;
+ }
+
+ void record_par_time(double ms) {
+ _cur_collection_par_time_ms = ms;
+ }
+
+ void record_aux_start_time(int i) {
+ guarantee(i < _aux_num, "should be within range");
+ _cur_aux_start_times_ms[i] = os::elapsedTime() * 1000.0;
+ }
+
+ void record_aux_end_time(int i) {
+ guarantee(i < _aux_num, "should be within range");
+ double ms = os::elapsedTime() * 1000.0 - _cur_aux_start_times_ms[i];
+ _cur_aux_times_set[i] = true;
+ _cur_aux_times_ms[i] += ms;
+ }
+
+ void record_pop_compute_rc_start();
+ void record_pop_compute_rc_end();
+
+ void record_pop_evac_start();
+ void record_pop_evac_end();
+
+ // Record the fact that "bytes" bytes allocated in a region.
+ void record_before_bytes(size_t bytes);
+ void record_after_bytes(size_t bytes);
+
+ // Returns "true" if this is a good time to do a collection pause.
+ // The "word_size" argument, if non-zero, indicates the size of an
+ // allocation request that is prompting this query.
+ virtual bool should_do_collection_pause(size_t word_size) = 0;
+
+ // Choose a new collection set. Marks the chosen regions as being
+ // "in_collection_set", and links them together. The head and number of
+ // the collection set are available via access methods.
+ // If "pop_region" is non-NULL, it is a popular region that has already
+ // been added to the collection set.
+ virtual void choose_collection_set(HeapRegion* pop_region = NULL) = 0;
+
+ void clear_collection_set() { _collection_set = NULL; }
+
+ // The head of the list (via "next_in_collection_set()") representing the
+ // current collection set.
+ HeapRegion* collection_set() { return _collection_set; }
+
+ // Sets the collection set to the given single region.
+ virtual void set_single_region_collection_set(HeapRegion* hr);
+
+ // The number of elements in the current collection set.
+ size_t collection_set_size() { return _collection_set_size; }
+
+ // Add "hr" to the CS.
+ void add_to_collection_set(HeapRegion* hr);
+
+ bool should_initiate_conc_mark() { return _should_initiate_conc_mark; }
+ void set_should_initiate_conc_mark() { _should_initiate_conc_mark = true; }
+ void unset_should_initiate_conc_mark(){ _should_initiate_conc_mark = false; }
+
+ void checkpoint_conc_overhead();
+
+ // If an expansion would be appropriate, because recent GC overhead had
+ // exceeded the desired limit, return an amount to expand by.
+ virtual size_t expansion_amount();
+
+ // note start of mark thread
+ void note_start_of_mark_thread();
+
+ // The marked bytes of the "r" has changed; reclassify it's desirability
+ // for marking. Also asserts that "r" is eligible for a CS.
+ virtual void note_change_in_marked_bytes(HeapRegion* r) = 0;
+
+#ifndef PRODUCT
+ // Check any appropriate marked bytes info, asserting false if
+ // something's wrong, else returning "true".
+ virtual bool assertMarkedBytesDataOK() = 0;
+#endif
+
+ // Print tracing information.
+ void print_tracing_info() const;
+
+ // Print stats on young survival ratio
+ void print_yg_surv_rate_info() const;
+
+ void finished_recalculating_age_indexes() {
+ _short_lived_surv_rate_group->finished_recalculating_age_indexes();
+ // do that for any other surv rate groups
+ }
+
+ bool should_add_next_region_to_young_list();
+
+ bool in_young_gc_mode() {
+ return _in_young_gc_mode;
+ }
+ void set_in_young_gc_mode(bool in_young_gc_mode) {
+ _in_young_gc_mode = in_young_gc_mode;
+ }
+
+ bool full_young_gcs() {
+ return _full_young_gcs;
+ }
+ void set_full_young_gcs(bool full_young_gcs) {
+ _full_young_gcs = full_young_gcs;
+ }
+
+ bool adaptive_young_list_length() {
+ return _adaptive_young_list_length;
+ }
+ void set_adaptive_young_list_length(bool adaptive_young_list_length) {
+ _adaptive_young_list_length = adaptive_young_list_length;
+ }
+
+ inline double get_gc_eff_factor() {
+ double ratio = _known_garbage_ratio;
+
+ double square = ratio * ratio;
+ // square = square * square;
+ double ret = square * 9.0 + 1.0;
+#if 0
+ gclog_or_tty->print_cr("ratio = %1.2lf, ret = %1.2lf", ratio, ret);
+#endif // 0
+ guarantee(0.0 <= ret && ret < 10.0, "invariant!");
+ return ret;
+ }
+
+ //
+ // Survivor regions policy.
+ //
+protected:
+
+ // Current tenuring threshold, set to 0 if the collector reaches the
+ // maximum amount of suvivors regions.
+ int _tenuring_threshold;
+
+public:
+
+ inline GCAllocPurpose
+ evacuation_destination(HeapRegion* src_region, int age, size_t word_sz) {
+ if (age < _tenuring_threshold && src_region->is_young()) {
+ return GCAllocForSurvived;
+ } else {
+ return GCAllocForTenured;
+ }
+ }
+
+ inline bool track_object_age(GCAllocPurpose purpose) {
+ return purpose == GCAllocForSurvived;
+ }
+
+ inline GCAllocPurpose alternative_purpose(int purpose) {
+ return GCAllocForTenured;
+ }
+
+ uint max_regions(int purpose);
+
+ // The limit on regions for a particular purpose is reached.
+ void note_alloc_region_limit_reached(int purpose) {
+ if (purpose == GCAllocForSurvived) {
+ _tenuring_threshold = 0;
+ }
+ }
+
+ void note_start_adding_survivor_regions() {
+ _survivor_surv_rate_group->start_adding_regions();
+ }
+
+ void note_stop_adding_survivor_regions() {
+ _survivor_surv_rate_group->stop_adding_regions();
+ }
+};
+
+// This encapsulates a particular strategy for a g1 Collector.
+//
+// Start a concurrent mark when our heap size is n bytes
+// greater then our heap size was at the last concurrent
+// mark. Where n is a function of the CMSTriggerRatio
+// and the MinHeapFreeRatio.
+//
+// Start a g1 collection pause when we have allocated the
+// average number of bytes currently being freed in
+// a collection, but only if it is at least one region
+// full
+//
+// Resize Heap based on desired
+// allocation space, where desired allocation space is
+// a function of survival rate and desired future to size.
+//
+// Choose collection set by first picking all older regions
+// which have a survival rate which beats our projected young
+// survival rate. Then fill out the number of needed regions
+// with young regions.
+
+class G1CollectorPolicy_BestRegionsFirst: public G1CollectorPolicy {
+ CollectionSetChooser* _collectionSetChooser;
+ // If the estimated is less then desirable, resize if possible.
+ void expand_if_possible(size_t numRegions);
+
+ virtual void choose_collection_set(HeapRegion* pop_region = NULL);
+ virtual void record_collection_pause_start(double start_time_sec,
+ size_t start_used);
+ virtual void record_concurrent_mark_cleanup_end(size_t freed_bytes,
+ size_t max_live_bytes);
+ virtual void record_full_collection_end();
+
+public:
+ G1CollectorPolicy_BestRegionsFirst() {
+ _collectionSetChooser = new CollectionSetChooser();
+ }
+ void record_collection_pause_end(bool popular, bool abandoned);
+ bool should_do_collection_pause(size_t word_size);
+ virtual void set_single_region_collection_set(HeapRegion* hr);
+ // This is not needed any more, after the CSet choosing code was
+ // changed to use the pause prediction work. But let's leave the
+ // hook in just in case.
+ void note_change_in_marked_bytes(HeapRegion* r) { }
+#ifndef PRODUCT
+ bool assertMarkedBytesDataOK();
+#endif
+};
+
+// This should move to some place more general...
+
+// If we have "n" measurements, and we've kept track of their "sum" and the
+// "sum_of_squares" of the measurements, this returns the variance of the
+// sequence.
+inline double variance(int n, double sum_of_squares, double sum) {
+ double n_d = (double)n;
+ double avg = sum/n_d;
+ return (sum_of_squares - 2.0 * avg * sum + n_d * avg * avg) / n_d;
+}
+
+// Local Variables: ***
+// c-indentation-style: gnu ***
+// End: ***
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.cpp
new file mode 100644
index 00000000000..74209dc3b79
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.cpp
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_g1MMUTracker.cpp.incl"
+
+#define _DISABLE_MMU 0
+
+// can't rely on comparing doubles with tolerating a small margin for error
+#define SMALL_MARGIN 0.0000001
+#define is_double_leq_0(_value) ( (_value) < SMALL_MARGIN )
+#define is_double_leq(_val1, _val2) is_double_leq_0((_val1) - (_val2))
+#define is_double_geq(_val1, _val2) is_double_leq_0((_val2) - (_val1))
+
+/***** ALL TIMES ARE IN SECS!!!!!!! *****/
+
+G1MMUTracker::G1MMUTracker(double time_slice, double max_gc_time) :
+ _time_slice(time_slice),
+ _max_gc_time(max_gc_time),
+ _conc_overhead_time_sec(0.0) { }
+
+void
+G1MMUTracker::update_conc_overhead(double conc_overhead) {
+ double conc_overhead_time_sec = _time_slice * conc_overhead;
+ if (conc_overhead_time_sec > 0.9 * _max_gc_time) {
+ // We are screwed, as we only seem to have <10% of the soft
+ // real-time goal available for pauses. Let's admit defeat and
+ // allow something more generous as a pause target.
+ conc_overhead_time_sec = 0.75 * _max_gc_time;
+ }
+
+ _conc_overhead_time_sec = conc_overhead_time_sec;
+}
+
+G1MMUTrackerQueue::G1MMUTrackerQueue(double time_slice, double max_gc_time) :
+ G1MMUTracker(time_slice, max_gc_time),
+ _head_index(0),
+ _tail_index(trim_index(_head_index+1)),
+ _no_entries(0) { }
+
+void G1MMUTrackerQueue::remove_expired_entries(double current_time) {
+ double limit = current_time - _time_slice;
+ while (_no_entries > 0) {
+ if (is_double_geq(limit, _array[_tail_index].end_time())) {
+ _tail_index = trim_index(_tail_index + 1);
+ --_no_entries;
+ } else
+ return;
+ }
+ guarantee(_no_entries == 0, "should have no entries in the array");
+}
+
+double G1MMUTrackerQueue::calculate_gc_time(double current_time) {
+ double gc_time = 0.0;
+ double limit = current_time - _time_slice;
+ for (int i = 0; i < _no_entries; ++i) {
+ int index = trim_index(_tail_index + i);
+ G1MMUTrackerQueueElem *elem = &_array[index];
+ if (elem->end_time() > limit) {
+ if (elem->start_time() > limit)
+ gc_time += elem->duration();
+ else
+ gc_time += elem->end_time() - limit;
+ }
+ }
+ return gc_time;
+}
+
+void G1MMUTrackerQueue::add_pause(double start, double end, bool gc_thread) {
+ double longest_allowed = longest_pause_internal(start);
+ if (longest_allowed < 0.0)
+ longest_allowed = 0.0;
+ double duration = end - start;
+
+ remove_expired_entries(end);
+ if (_no_entries == QueueLength) {
+ // OK, right now when we fill up we bomb out
+ // there are a few ways of dealing with this "gracefully"
+ // increase the array size (:-)
+ // remove the oldest entry (this might allow more GC time for
+ // the time slice than what's allowed)
+ // concolidate the two entries with the minimum gap between them
+ // (this mighte allow less GC time than what's allowed)
+ guarantee(0, "array full, currently we can't recover");
+ }
+ _head_index = trim_index(_head_index + 1);
+ ++_no_entries;
+ _array[_head_index] = G1MMUTrackerQueueElem(start, end);
+}
+
+// basically the _internal call does not remove expired entries
+// this is for trying things out in the future and a couple
+// of other places (debugging)
+
+double G1MMUTrackerQueue::longest_pause(double current_time) {
+ if (_DISABLE_MMU)
+ return _max_gc_time;
+
+ MutexLockerEx x(MMUTracker_lock, Mutex::_no_safepoint_check_flag);
+ remove_expired_entries(current_time);
+
+ return longest_pause_internal(current_time);
+}
+
+double G1MMUTrackerQueue::longest_pause_internal(double current_time) {
+ double target_time = _max_gc_time;
+
+ while( 1 ) {
+ double gc_time =
+ calculate_gc_time(current_time + target_time) + _conc_overhead_time_sec;
+ double diff = target_time + gc_time - _max_gc_time;
+ if (!is_double_leq_0(diff)) {
+ target_time -= diff;
+ if (is_double_leq_0(target_time)) {
+ target_time = -1.0;
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+
+ return target_time;
+}
+
+// basically the _internal call does not remove expired entries
+// this is for trying things out in the future and a couple
+// of other places (debugging)
+
+double G1MMUTrackerQueue::when_sec(double current_time, double pause_time) {
+ if (_DISABLE_MMU)
+ return 0.0;
+
+ MutexLockerEx x(MMUTracker_lock, Mutex::_no_safepoint_check_flag);
+ remove_expired_entries(current_time);
+
+ return when_internal(current_time, pause_time);
+}
+
+double G1MMUTrackerQueue::when_internal(double current_time,
+ double pause_time) {
+ // if the pause is over the maximum, just assume that it's the maximum
+ double adjusted_pause_time =
+ (pause_time > max_gc_time()) ? max_gc_time() : pause_time;
+ double earliest_end = current_time + adjusted_pause_time;
+ double limit = earliest_end - _time_slice;
+ double gc_time = calculate_gc_time(earliest_end);
+ double diff = gc_time + adjusted_pause_time - max_gc_time();
+ if (is_double_leq_0(diff))
+ return 0.0;
+
+ int index = _tail_index;
+ while ( 1 ) {
+ G1MMUTrackerQueueElem *elem = &_array[index];
+ if (elem->end_time() > limit) {
+ if (elem->start_time() > limit)
+ diff -= elem->duration();
+ else
+ diff -= elem->end_time() - limit;
+ if (is_double_leq_0(diff))
+ return elem->end_time() + diff + _time_slice - adjusted_pause_time - current_time;
+ }
+ index = trim_index(index+1);
+ guarantee(index != trim_index(_head_index + 1), "should not go past head");
+ }
+}
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp b/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp
new file mode 100644
index 00000000000..88a3707626b
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1MMUTracker.hpp
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2001-2007 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+// Keeps track of the GC work and decides when it is OK to do GC work
+// and for how long so that the MMU invariants are maintained.
+
+/***** ALL TIMES ARE IN SECS!!!!!!! *****/
+
+// this is the "interface"
+class G1MMUTracker {
+protected:
+ double _time_slice;
+ double _max_gc_time; // this is per time slice
+
+ double _conc_overhead_time_sec;
+
+public:
+ G1MMUTracker(double time_slice, double max_gc_time);
+
+ void update_conc_overhead(double conc_overhead);
+
+ virtual void add_pause(double start, double end, bool gc_thread) = 0;
+ virtual double longest_pause(double current_time) = 0;
+ virtual double when_sec(double current_time, double pause_time) = 0;
+
+ double max_gc_time() {
+ return _max_gc_time - _conc_overhead_time_sec;
+ }
+
+ inline bool now_max_gc(double current_time) {
+ return when_sec(current_time, max_gc_time()) < 0.00001;
+ }
+
+ inline double when_max_gc_sec(double current_time) {
+ return when_sec(current_time, max_gc_time());
+ }
+
+ inline jlong when_max_gc_ms(double current_time) {
+ double when = when_max_gc_sec(current_time);
+ return (jlong) (when * 1000.0);
+ }
+
+ inline jlong when_ms(double current_time, double pause_time) {
+ double when = when_sec(current_time, pause_time);
+ return (jlong) (when * 1000.0);
+ }
+};
+
+class G1MMUTrackerQueueElem {
+private:
+ double _start_time;
+ double _end_time;
+
+public:
+ inline double start_time() { return _start_time; }
+ inline double end_time() { return _end_time; }
+ inline double duration() { return _end_time - _start_time; }
+
+ G1MMUTrackerQueueElem() {
+ _start_time = 0.0;
+ _end_time = 0.0;
+ }
+
+ G1MMUTrackerQueueElem(double start_time, double end_time) {
+ _start_time = start_time;
+ _end_time = end_time;
+ }
+};
+
+// this is an implementation of the MMUTracker using a (fixed-size) queue
+// that keeps track of all the recent pause times
+class G1MMUTrackerQueue: public G1MMUTracker {
+private:
+ enum PrivateConstants {
+ QueueLength = 64
+ };
+
+ // The array keeps track of all the pauses that fall within a time
+ // slice (the last time slice during which pauses took place).
+ // The data structure implemented is a circular queue.
+ // Head "points" to the most recent addition, tail to the oldest one.
+ // The array is of fixed size and I don't think we'll need more than
+ // two or three entries with the current behaviour of G1 pauses.
+ // If the array is full, an easy fix is to look for the pauses with
+ // the shortest gap between them and concolidate them.
+
+ G1MMUTrackerQueueElem _array[QueueLength];
+ int _head_index;
+ int _tail_index;
+ int _no_entries;
+
+ inline int trim_index(int index) {
+ return (index + QueueLength) % QueueLength;
+ }
+
+ void remove_expired_entries(double current_time);
+ double calculate_gc_time(double current_time);
+
+ double longest_pause_internal(double current_time);
+ double when_internal(double current_time, double pause_time);
+
+public:
+ G1MMUTrackerQueue(double time_slice, double max_gc_time);
+
+ virtual void add_pause(double start, double end, bool gc_thread);
+
+ virtual double longest_pause(double current_time);
+ virtual double when_sec(double current_time, double pause_time);
+};
diff --git a/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp b/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp
new file mode 100644
index 00000000000..42d177a1e59
--- /dev/null
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp
@@ -0,0 +1,375 @@
+/*
+ * Copyright 2001-2008 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+#include "incls/_precompiled.incl"
+#include "incls/_g1MarkSweep.cpp.incl"
+
+class HeapRegion;
+
+void G1MarkSweep::invoke_at_safepoint(ReferenceProcessor* rp,
+ bool clear_all_softrefs) {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
+
+ // hook up weak ref data so it can be used during Mark-Sweep
+ assert(GenMarkSweep::ref_processor() == NULL, "no stomping");
+ assert(rp != NULL, "should be non-NULL");
+ GenMarkSweep::_ref_processor = rp;
+ rp->setup_policy(clear_all_softrefs);
+
+ // When collecting the permanent generation methodOops may be moving,
+ // so we either have to flush all bcp data or convert it into bci.
+ CodeCache::gc_prologue();
+ Threads::gc_prologue();
+
+ // Increment the invocation count for the permanent generation, since it is
+ // implicitly collected whenever we do a full mark sweep collection.
+ SharedHeap* sh = SharedHeap::heap();
+ sh->perm_gen()->stat_record()->invocations++;
+
+ bool marked_for_unloading = false;
+
+ allocate_stacks();
+
+ // We should save the marks of the currently locked biased monitors.
+ // The marking doesn't preserve the marks of biased objects.
+ BiasedLocking::preserve_marks();
+
+ mark_sweep_phase1(marked_for_unloading, clear_all_softrefs);
+
+ if (G1VerifyConcMark) {
+ G1CollectedHeap* g1h = G1CollectedHeap::heap();
+ g1h->checkConcurrentMark();
+ }
+
+ mark_sweep_phase2();
+
+ // Don't add any more derived pointers during phase3
+ COMPILER2_PRESENT(DerivedPointerTable::set_active(false));
+
+ mark_sweep_phase3();
+
+ mark_sweep_phase4();
+
+ GenMarkSweep::restore_marks();
+ BiasedLocking::restore_marks();
+ GenMarkSweep::deallocate_stacks();
+
+ // We must invalidate the perm-gen rs, so that it gets rebuilt.
+ GenRemSet* rs = sh->rem_set();
+ rs->invalidate(sh->perm_gen()->used_region(), true /*whole_heap*/);
+
+ // "free at last gc" is calculated from these.
+ // CHF: cheating for now!!!
+ // Universe::set_heap_capacity_at_last_gc(Universe::heap()->capacity());
+ // Universe::set_heap_used_at_last_gc(Universe::heap()->used());
+
+ Threads::gc_epilogue();
+ CodeCache::gc_epilogue();
+
+ // refs processing: clean slate
+ GenMarkSweep::_ref_processor = NULL;
+}
+
+
+void G1MarkSweep::allocate_stacks() {
+ GenMarkSweep::_preserved_count_max = 0;
+ GenMarkSweep::_preserved_marks = NULL;
+ GenMarkSweep::_preserved_count = 0;
+ GenMarkSweep::_preserved_mark_stack = NULL;
+ GenMarkSweep::_preserved_oop_stack = NULL;
+
+ GenMarkSweep::_marking_stack =
+ new (ResourceObj::C_HEAP) GrowableArray(4000, true);
+
+ size_t size = SystemDictionary::number_of_classes() * 2;
+ GenMarkSweep::_revisit_klass_stack =
+ new (ResourceObj::C_HEAP) GrowableArray