8044102: Ensure bechmark exclude list for Octane benchmarks is in only one place, project.properties, and fix benchmark harness

Reviewed-by: attila, sundar
This commit is contained in:
Marcus Lagergren 2014-05-28 13:58:46 +02:00
parent a6e90a4ba3
commit d4287c2288
5 changed files with 181 additions and 125 deletions

@ -22,15 +22,16 @@
# questions.
#
#convert tabs to spaces
find . -name "*.java" -exec sed -i "" 's/ / /g' {} \;
#remove trailing whitespace
find . -name "*.java" -exec sed -i "" 's/[ ]*$//' \{} \;
#convert tabs to spaces
find . -name "*.js" -exec sed -i "" 's/ / /g' {} \;
#remove trailing whitespace
find . -name "*.js" -exec sed -i "" 's/[ ]*$//' \{} \;
fix() {
#convert tabs to spaces
find . -name $1 -exec sed -i "" 's/ / /g' {} \;
#remove trailing whitespace
find . -name $1 -exec sed -i "" 's/[ ]*$//' \{} \;
}
if [ ! -z $1 ]; then
fix $1;
else
fix "*.java"
fix "*.js"
fi

@ -7,40 +7,41 @@
This code is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License version 2 only, as
published by the Free Software Foundation.
This code is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
version 2 for more details (a copy is included in the LICENSE file that
accompanied this code).
You should have received a copy of the GNU General Public License version
2 along with this work; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
or visit www.oracle.com if you need additional information or have any
questions.
-->
<project
name="nashorn-benchmarks"
default="all"
basedir=".."
<project
name="nashorn-benchmarks"
default="all"
basedir=".."
xmlns:if="ant:if">
<!--
Below are the octane benchmarks that should be run.
<!--
Below are the octane benchmarks that should be run.
The ones that are excluded, as Nashorn currently has
some issues with them (functionality or performance)
some issues with them (functionality or performance)
are commented out
-->
<!-- box2d -->
<target name="octane-box2d" depends="jar">
<target name="octane-box2d" depends="octane-box2d-nashorn"/>
<target name="octane-box2d-nashorn" depends="jar">
<run-one cond="octane.benchmark.box2d" runtime="nashorn"/>
</target>
</target>
<target name="octane-box2d-v8" depends="jar">
<run-one cond="octane.benchmark.box2d" runtime="v8"/>
</target>
@ -48,10 +49,11 @@
<run-one cond="octane.benchmark.box2d" runtime="rhino"/>
</target>
<!-- code-load -->
<target name="octane-code-load" depends="jar">
<!-- code-load -->
<target name="octane-code-load" depends="octane-code-load-nashorn"/>
<target name="octane-code-load-nashorn" depends="jar">
<run-one cond="octane.benchmark.code-load" runtime="nashorn"/>
</target>
</target>
<target name="octane-code-load-v8" depends="jar">
<run-one cond="octane.benchmark.code-load" runtime="v8"/>
</target>
@ -60,9 +62,10 @@
</target>
<!-- crypto -->
<target name="octane-crypto" depends="jar">
<target name="octane-crypto" depends="octane-crypto-nashorn"/>
<target name="octane-crypto-nashorn" depends="jar">
<run-one cond="octane.benchmark.crypto" runtime="nashorn"/>
</target>
</target>
<target name="octane-crypto-v8" depends="jar">
<run-one cond="octane.benchmark.crypto" runtime="v8"/>
</target>
@ -71,9 +74,10 @@
</target>
<!-- deltablue -->
<target name="octane-deltablue" depends="jar">
<target name="octane-deltablue" depends="octane-deltablue-nashorn"/>
<target name="octane-deltablue-nashorn" depends="jar">
<run-one cond="octane.benchmark.deltablue" runtime="nashorn"/>
</target>
</target>
<target name="octane-deltablue-v8" depends="jar">
<run-one cond="octane.benchmark.deltablue" runtime="v8"/>
</target>
@ -82,20 +86,22 @@
</target>
<!-- earley-boyer -->
<target name="octane-earley-boyer" depends="jar">
<target name="octane-earley-boyer" depends="octane-earley-boyer-nashorn"/>
<target name="octane-earley-boyer-nashorn" depends="jar">
<run-one cond="octane.benchmark.earley-boyer" runtime="nashorn"/>
</target>
</target>
<target name="octane-earley-boyer-v8" depends="jar">
<run-one cond="octane.benchmark.earley-boyer" runtime="v8"/>
</target>
<target name="octane-earley-boyer-rhino" depends="jar">
<run-one cond="octane.benchmark.earley-boyer" runtime="rhino"/>
</target>
<!-- gbemu -->
<target name="octane-gbemu" depends="jar">
<!-- gbemu -->
<target name="octane-gbemu" depends="octane-gbemu-nashorn"/>
<target name="octane-gbemu-nashorn" depends="jar">
<run-one cond="octane.benchmark.gbemu" runtime="nashorn"/>
</target>
</target>
<target name="octane-gbemu-v8" depends="jar">
<run-one cond="octane.benchmark.gbemu" runtime="v8"/>
</target>
@ -103,10 +109,11 @@
<run-one cond="octane.benchmark.gbemu" runtime="rhino"/>
</target>
<!-- mandreel -->
<target name="octane-mandreel" depends="jar">
<!-- mandreel -->
<target name="octane-mandreel" depends="octane-mandreel-nashorn"/>
<target name="octane-mandreel-nashorn" depends="jar">
<run-one cond="octane.benchmark.mandreel" runtime="nashorn"/>
</target>
</target>
<target name="octane-mandreel-v8" depends="jar">
<run-one cond="octane.benchmark.mandreel" runtime="v8"/>
</target>
@ -115,9 +122,10 @@
</target>
<!-- navier-stokes -->
<target name="octane-navier-stokes" depends="jar">
<target name="octane-navier-stokes" depends="octane-navier-stokes-nashorn"/>
<target name="octane-navier-stokes-nashorn" depends="jar">
<run-one cond="octane.benchmark.navier-stokes" runtime="nashorn"/>
</target>
</target>
<target name="octane-navier-stokes-v8" depends="jar">
<run-one cond="octane.benchmark.navier-stokes" runtime="v8"/>
</target>
@ -126,9 +134,10 @@
</target>
<!-- pdfjs -->
<target name="octane-pdfjs" depends="jar">
<target name="octane-pdfjs" depends="octane-pdfjs-nashorn"/>
<target name="octane-pdfjs-nashorn" depends="jar">
<run-one cond="octane.benchmark.pdfjs" runtime="nashorn"/>
</target>
</target>
<target name="octane-pdfjs-v8" depends="jar">
<run-one cond="octane.benchmark.pdfjs" runtime="v8"/>
</target>
@ -137,9 +146,10 @@
</target>
<!-- raytrace -->
<target name="octane-raytrace" depends="jar">
<target name="octane-raytrace" depends="octane-raytrace-nashorn"/>
<target name="octane-raytrace-nashorn" depends="jar">
<run-one cond="octane.benchmark.raytrace" runtime="nashorn"/>
</target>
</target>
<target name="octane-raytrace-v8" depends="jar">
<run-one cond="octane.benchmark.raytrace" runtime="v8"/>
</target>
@ -148,9 +158,10 @@
</target>
<!-- regexp -->
<target name="octane-regexp" depends="jar">
<target name="octane-regexp" depends="octane-regexp-nashorn"/>
<target name="octane-regexp-nashorn" depends="jar">
<run-one cond="octane.benchmark.regexp" runtime="nashorn"/>
</target>
</target>
<target name="octane-regexp-v8" depends="jar">
<run-one cond="octane.benchmark.regexp" runtime="v8"/>
</target>
@ -159,9 +170,10 @@
</target>
<!-- richards -->
<target name="octane-richards" depends="jar">
<target name="octane-richards" depends="octane-richards-nashorn"/>
<target name="octane-richards-nashorn" depends="jar">
<run-one cond="octane.benchmark.richards" runtime="nashorn"/>
</target>
</target>
<target name="octane-richards-v8" depends="jar">
<run-one cond="octane.benchmark.richards" runtime="v8"/>
</target>
@ -170,9 +182,10 @@
</target>
<!-- splay -->
<target name="octane-splay" depends="jar">
<target name="octane-splay" depends="octane-splay-nashorn"/>
<target name="octane-splay-nashorn" depends="jar">
<run-one cond="octane.benchmark.splay" runtime="nashorn"/>
</target>
</target>
<target name="octane-splay-v8" depends="jar">
<run-one cond="octane.benchmark.splay" runtime="v8"/>
</target>
@ -181,9 +194,10 @@
</target>
<!-- typescript -->
<target name="octane-typescript" depends="jar">
<target name="octane-typescript" depends="octane-typescript-nashorn"/>
<target name="octane-typescript-nashorn" depends="jar">
<run-one cond="octane.benchmark.typescript" runtime="nashorn"/>
</target>
</target>
<target name="octane-typescript-v8" depends="jar">
<run-one cond="octane.benchmark.typescript" runtime="v8"/>
</target>
@ -192,9 +206,10 @@
</target>
<!-- zlib -->
<target name="octane-zlib" depends="jar">
<target name="octane-zlib" depends="octane-zlib-nashorn"/>
<target name="octane-zlib-nashorn" depends="jar">
<run-one cond="octane.benchmark.zlib" runtime="nashorn"/>
</target>
</target>
<target name="octane-zlib-v8" depends="jar">
<run-one cond="octane.benchmark.zlib" runtime="v8"/>
</target>
@ -209,76 +224,92 @@
<target name="octane-process-separate" if="${octane-test-sys-prop.separate.process}">
<echo message="Running each benchmark in separate processes, starting new JVMs for each."/>
<!-- TODO: possibly turn this into a loop with ant.contrib -->
<run-one cond="octane.benchmark.box2d" runtime="${runtime}"/>
<!--run-one cond="octane.benchmark.crypto" runtime="${runtime}"/>-->
<run-one cond="octane.benchmark.code-load" runtime="${runtime}"/>
<run-one cond="octane.benchmark.deltablue" runtime="${runtime}"/>
<run-one cond="octane.benchmark.earley-boyer" runtime="${runtime}"/>
<run-one cond="octane.benchmark.gbemu" runtime="${runtime}"/>
<run-one cond="octane.benchmark.navier-stokes" runtime="${runtime}"/>
<run-one cond="octane.benchmark.mandreel" runtime="${runtime}"/>
<run-one cond="octane.benchmark.pdfjs" runtime="${runtime}"/>
<run-one cond="octane.benchmark.raytrace" runtime="${runtime}"/>
<run-one cond="octane.benchmark.regexp" runtime="${runtime}"/>
<run-one cond="octane.benchmark.richards" runtime="${runtime}"/>
<run-one cond="octane.benchmark.splay" runtime="${runtime}"/>
<!--<run-one cond="octane.benchmark.typescript" runtime="${runtime}"/>-->
<!--<run-one cond="octane.benchmark.zlib" runtime="${runtime}"/>-->
<script language="javascript"><![CDATA[
var props = [];
for (var prop in project.getProperties()) {
if (prop.startsWith("octane.benchmark.")) {
props.push(prop);
}
}
//sort benchmark props in alphabetical order by name
props.sort(function(a, b) {
if (a < b) {
return -1;
} else if (a > b) {
return 1;
} else {
return 0;
}
});
var runtime = project.getProperty("runtime");
for (var i in props) {
var task = project.createTask("run-one");
// workaround for https://issues.apache.org/bugzilla/show_bug.cgi?id=53831, still not fixed
if (task.getOwningTarget() == null) {
task.setOwningTarget(self.getOwningTarget());
}
var prop = props[i];
task.setDynamicAttribute("cond", prop);
task.setDynamicAttribute("runtime", runtime);
task.perform();
}
]]></script>
</target>
<target name="octane-process-single" unless="${octane-test-sys-prop.separate.process}">
<target name="octane-process-single" unless="${octane-test-sys-prop.separate.process}">
<echo message="Running all benchmarks in the same process."/>
<echo message="multiple -- ${octane.benchmarks}"/>
<pathconvert property="octane.benchmarks" pathsep=" ">
<propertyset>
<propertyref prefix="octane.benchmark."/>
<propertyref prefix="octane.benchmark."/>
</propertyset>
</pathconvert>
<echo message="monkey= ${octane.benchmarks}"/>
<antcall target="run-octane${runtime}">
<param name="octane-tests" value="${octane.benchmarks}"/>
</antcall>
</antcall>
</target>
<!--
run 'octane' in single or separate processes based on config
<!--
run 'octane' in single or separate processes based on config
This uses nashorn as the default runtime
-->
<target name="octane-nashorn" depends="jar">
<property name="runtime" value="nashorn"/>
<property name="runtime" value="nashorn"/>
<antcall target="octane-process-separate"/>
<antcall target="octane-process-single"/>
</target>
<!-- alias for 'octane' -->
<target name="octane" depends="octane-nashorn"/>
<!-- run octane benchmarks using octane as runtime -->
<target name="octane-v8" depends="jar">
<property name="runtime" value="v8"/>
<property name="runtime" value="v8"/>
<antcall target="octane-process-separate"/>
<antcall target="octane-process-single"/>
</target>
<!-- run octane benchmarks using Rhino as runtime -->
<target name="octane-rhino" depends="jar">
<property name="runtime" value="rhino"/>
<property name="runtime" value="rhino"/>
<antcall target="octane-process-separate"/>
<antcall target="octane-process-single"/>
</target>
<macrodef name="run-one">
<attribute name="cond"/>
<attribute name="runtime" default=""/>
<sequential>
<antcall target="run-octane-@{runtime}" if:set="@{cond}">
<param name="octane-tests" value="${@{cond}}"/>
</antcall>
<antcall target="run-octane-@{runtime}" if:set="@{cond}">
<param name="octane-tests" value="${@{cond}}"/>
</antcall>
</sequential>
</macrodef>
<target name="run-octane-nashorn" if="octane-tests">
<target name="run-octane-nashorn">
<java classname="${nashorn.shell.tool}"
classpath="${run.test.classpath}"
fork="true"
@ -304,7 +335,7 @@
<exec executable="${v8.shell}">
<arg value="${octane-test-sys-prop.test.js.framework}"/>
<arg value="--"/>
<arg value="${octane-tests}"/>
<arg value="${octane-tests}"/>
<arg value="--runtime"/>
<arg value="v8"/>
<arg value="--verbose"/>
@ -336,11 +367,11 @@
<arg value="${octane-tests}/"/>
</exec>
</target>
<target name="sunspider-init" depends="jar">
<fileset id="sunspider-set"
dir="${sunspider-test-sys-prop.test.js.roots}"
excludes="${sunspider-test-sys-prop.test.js.exclude.list}">
dir="${sunspider-test-sys-prop.test.js.roots}"
excludes="${sunspider-test-sys-prop.test.js.exclude.list}">
<include name="**/*.js"/>
</fileset>
<pathconvert pathsep=" " property="sunspider-tests" refid="sunspider-set"/>
@ -349,7 +380,9 @@
<!--- SUNSPIDER JOB BELOW -->
<!-- run sunspider with Nashorn -->
<target name="sunspider" depends="sunspider-init">
<target name="sunspider" depends="sunspider-nashorn"/>
<target name="sunspider-nashorn" depends="sunspider-init">
<java classname="${nashorn.shell.tool}"
classpath="${run.test.classpath}"
fork="true"
@ -361,6 +394,9 @@
<arg value="${sunspider-test-sys-prop.test.js.framework}"/>
<arg value="--"/>
<arg value="${sunspider-tests}/"/>
<arg value="--verbose"/>
<arg value="--times"/>
<arg value="${sunspider.iterations}"/>
</java>
</target>
@ -370,6 +406,9 @@
<arg value="${sunspider-test-sys-prop.test.js.framework}"/>
<arg value="--"/>
<arg value="${sunspider-tests}/"/>
<arg value="--verbose"/>
<arg value="--times"/>
<arg value="${sunspider.iterations}"/>
</exec>
</target>
@ -384,6 +423,9 @@
<arg value="9"/>
<arg value="${sunspider-test-sys-prop.test.js.framework}"/>
<arg value="${sunspider-tests}/"/>
<arg value="--verbose"/>
<arg value="--times"/>
<arg value="${sunspider.iterations}"/>
</java>
</target>

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@ -21,6 +22,7 @@
or visit www.oracle.com if you need additional information or have any
questions.
-->
<project name="nashorn" default="test" basedir="..">
<import file="build-nasgen.xml"/>
<import file="code_coverage.xml"/>
@ -46,7 +48,7 @@
<available property="testng.available" file="${file.reference.testng.jar}"/>
<!-- check if Jemmy ang testng.jar are avaiable -->
<condition property="jemmy.jfx.testng.available" value="true">
<and>
<and>
<available file="${file.reference.jemmyfx.jar}"/>
<available file="${file.reference.jemmycore.jar}"/>
<available file="${file.reference.jemmyawtinput.jar}"/>
@ -381,15 +383,15 @@ grant codeBase "file:/${basedir}/test/script/markdown.js" {
<fileset id="test.classes" dir="${build.test.classes.dir}">
<include name="**/framework/*Test.class"/>
</fileset>
<copy file="${file.reference.jfxrt.jar}" todir="dist"/>
<condition property="jfx.prism.order" value="-Dprism.order=j2d" else=" ">
<not>
<not>
<os family="mac"/>
</not>
</condition>
</condition>
<testng outputdir="${build.test.results.dir}" classfilesetref="test.classes"
verbose="${testng.verbose}" haltonfailure="true" useDefaultListeners="false" listeners="${testng.listeners}" workingDir="${basedir}">
<jvmarg line="${ext.class.path}"/>
@ -404,7 +406,7 @@ grant codeBase "file:/${basedir}/test/script/markdown.js" {
</classpath>
</testng>
</target>
<target name="testmarkdown" depends="jar, check-testng, check-external-tests, compile-test, generate-policy-file" if="testng.available">
<fileset id="test.classes" dir="${build.test.classes.dir}">
<include name="**/framework/*Test.class"/>
@ -423,7 +425,7 @@ grant codeBase "file:/${basedir}/test/script/markdown.js" {
</classpath>
</testng>
</target>
<target name="test262" depends="jar, check-testng, check-external-tests, compile-test, generate-policy-file" if="testng.available">
<fileset id="test.classes" dir="${build.test.classes.dir}">
<include name="**/framework/*Test.class"/>
@ -463,20 +465,18 @@ grant codeBase "file:/${basedir}/test/script/markdown.js" {
</java>
</target>
<!-- classpath="${build.test.classes.dir}"-->
<target name="testparallel" depends="test-parallel"/>
<target name="test-parallel" depends="jar, check-testng, check-external-tests, compile-test, generate-policy-file" if="testng.available">
<!-- use just build.test.classes.dir to avoid referring to TestNG -->
<java classname="${parallel.test.runner}" dir="${basedir}"
failonerror="true"
fork="true">
failonerror="true"
fork="true">
<jvmarg line="${ext.class.path}"/>
<jvmarg line="${run.test.jvmargs} -Xmx${run.test.xmx} ${run.test.jvmsecurityargs}"/>
<classpath>
<pathelement path="${run.test.classpath}"/>
<pathelement path="${build.test.classes.dir}"/>
<pathelement path="${build.test.classes.dir}"/>
</classpath>
<syspropertyset>
<propertyref prefix="test-sys-prop."/>
@ -593,7 +593,7 @@ grant codeBase "file:/${basedir}/test/script/markdown.js" {
<mkdir dir="${test.external.dir}/yui"/>
<get src="http://yui.yahooapis.com/3.5.1/build/yui/yui.js" dest="${test.external.dir}/yui" skipexisting="true" ignoreerrors="true"/>
<get src="http://yui.yahooapis.com/3.5.1/build/yui/yui-min.js" dest="${test.external.dir}/yui" skipexisting="true" ignoreerrors="true"/>
<!-- showdown -->
<mkdir dir="${test.external.dir}/showdown"/>
<get src="https://raw.github.com/coreyti/showdown/master/src/showdown.js" dest="${test.external.dir}/showdown" skipexisting="true" ignoreerrors="true"/>

@ -1,5 +1,5 @@
#
# Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2010, 2014, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@ -185,6 +185,7 @@ test262-test-sys-prop.test.js.shared.context=true
# test262 test root
test262-test-sys-prop.test.js.roots=${test262.suite.dir}
# test262 enable/disable strict mode tests
test262-test-sys-prop.test.js.enable.strict.mode=true
@ -305,7 +306,7 @@ run.test.jvmsecurityargs=-Xverify:all -Djava.security.manager -Djava.security.po
test-sys-prop.test.fork.jvm.options=${run.test.jvmargs.main} -Xmx${run.test.xmx} ${run.test.jvmsecurityargs} -cp ${run.test.classpath}
# path of rhino.jar for benchmarks
rhino.dir=/Users/marcus/src/rhino
rhino.dir=
rhino.jar=${rhino.dir}/js.jar
v8.shell=d8
@ -343,19 +344,25 @@ octaneperf-sys-prop.rhino.jar=${rhino.jar}
#timeout for performance tests in minutes
octaneperf-sys-prop.timeout.value=10
#how many iterations to run sunspider after warmup
sunspider.iterations=3000
#################
# code coverage #
#################
#enable/disable code coverage; please redifine in the ${user.home}/.nashorn.project.local.properties
#enable/disable code coverage; please redifine in the ${user.home}/.nashorn.project.local.properties
make.code.coverage=false
#type of codecoverage; one of static or dynamic. Now only dynamic is supported
#type of codecoverage; one of static or dynamic. Now only dynamic is supported
jcov=dynamic
#naming of CC results
#NB directory specified in the cc.dir will be cleaned up!!!
#naming of CC results
#NB directory specified in the cc.dir will be cleaned up!!!
cc.dir=${basedir}/../Codecoverage_Nashorn
cc.result.file.name=CC_${jcov}_nashorn.xml
#dynamic CC parameters; please redefine in the ${user.home}/.nashorn.project.local.properties
#dynamic CC parameters; please redefine in the ${user.home}/.nashorn.project.local.properties
jcov2.lib.dir=${basedir}/../jcov2/lib
jcov.jar=${jcov2.lib.dir}/jcov.jar
cc.include=jdk\.nashorn\.*

@ -127,26 +127,32 @@ function runsuite(tests) {
assertEq(tests[n].actual(), tests[n].expected());
}
var times = 0;
if (typeof tests[n].rerun !== 'undefined' && tests[n].times > 0) {
pprint("rerunning " + tests[n].name + " " + tests[n].times + " times...");
var times = 0;
var to = tests[n].times;
var elemsPerPercent = to / 100;
var po = 0|(to / 10);
times = 0;
for (; times < to; times++) {
pprint("Doing warmup.");
for (times = 0; times < to; times++) {
initrandom();
tests[n].rerun();
}
pprint("Doing hot runs.");
for (times = 0; times < to; times++) {
initrandom();
tests[n].rerun();
if ((times % (po|0)) == 0) {
pprint(times/to * 100 + "%");
pprint("\t" + times/to * 100 + "%");
}
}
}
var t = new Date - dd;
pprint("time: " + t + " ms");
var t = Math.round(((new Date - dd) / (times == 0 ? 1 : times)) * 100 / 100);
pprint("time per iteration: " + t + " ms");
if (typeof tests[n].actual !== 'undefined') {
assertEq(tests[n].actual(), tests[n].expected());
}
@ -156,7 +162,7 @@ function runsuite(tests) {
changed = true;
} catch(e) {
if(runall) {
if (runall) {
print("FAIL!");
} else {
throw e;
@ -174,7 +180,7 @@ function runsuite(tests) {
for (var n = 0; n < tests.length; n++) {
var time = "" + res[n];
while (time.length < 4) {
while (time.length < 6) {
time = " " + time;
}
time += " ms";