8198318: Make build comparisons clean again

Reviewed-by: tbell, erikj

make/Images.gmk

@@ -134,18 +134,22 @@ $(JDK_IMAGE_DIR)/$(JIMAGE_TARGET_FILE): $(JMODS) \
     $(call DependOnVariable, JDK_MODULES_LIST) $(BASE_RELEASE_FILE)
 	$(ECHO) Creating jdk jimage
 	$(RM) -r $(JDK_IMAGE_DIR)
-	$(JLINK_TOOL) --add-modules $(JDK_MODULES_LIST) \
+	$(call ExecuteWithLog, $(SUPPORT_OUTPUTDIR)/images/jdk, \
-	    $(JLINK_JDK_EXTRA_OPTS) \
+	    $(JLINK_TOOL) --add-modules $(JDK_MODULES_LIST) \
-	    --output $(JDK_IMAGE_DIR)
+	        $(JLINK_JDK_EXTRA_OPTS) \
+	        --output $(JDK_IMAGE_DIR) \
+	)
 	$(TOUCH) $@
 
 $(JRE_IMAGE_DIR)/$(JIMAGE_TARGET_FILE): $(JMODS) \
     $(call DependOnVariable, JRE_MODULES_LIST) $(BASE_RELEASE_FILE)
 	$(ECHO) Creating jre jimage
 	$(RM) -r $(JRE_IMAGE_DIR)
-	$(JLINK_TOOL) --add-modules $(JRE_MODULES_LIST) \
+	$(call ExecuteWithLog, $(SUPPORT_OUTPUTDIR)/images/jre, \
-	    $(JLINK_JRE_EXTRA_OPTS) \
+	    $(JLINK_TOOL) --add-modules $(JRE_MODULES_LIST) \
-	    --output $(JRE_IMAGE_DIR)
+	        $(JLINK_JRE_EXTRA_OPTS) \
+	        --output $(JRE_IMAGE_DIR) \
+	)
 	$(TOUCH) $@
 
 
@@ -153,27 +157,33 @@ $(JRE_COMPACT1_IMAGE_DIR)/$(JIMAGE_TARGET_FILE): $(JMODS) \
     $(call DependOnVariable, JRE_COMPACT1_MODULES_LIST) $(BASE_RELEASE_FILE)
 	$(ECHO) Creating jre compact1 jimage
 	$(RM) -r $(JRE_COMPACT1_IMAGE_DIR)
-	$(JLINK_TOOL) --add-modules $(JRE_COMPACT1_MODULES_LIST) \
+	$(call ExecuteWithLog, $(SUPPORT_OUTPUTDIR)/images/jre_compact1, \
-	    $(JLINK_JRE_EXTRA_OPTS) \
+	    $(JLINK_TOOL) --add-modules $(JRE_COMPACT1_MODULES_LIST) \
-	    --output $(JRE_COMPACT1_IMAGE_DIR)
+	        $(JLINK_JRE_EXTRA_OPTS) \
+	        --output $(JRE_COMPACT1_IMAGE_DIR) \
+	)
 	$(TOUCH) $@
 
 $(JRE_COMPACT2_IMAGE_DIR)/$(JIMAGE_TARGET_FILE): $(JMODS) \
     $(call DependOnVariable, JRE_COMPACT2_MODULES_LIST) $(BASE_RELEASE_FILE)
 	$(ECHO) Creating jre compact2 jimage
 	$(RM) -r $(JRE_COMPACT2_IMAGE_DIR)
-	$(JLINK_TOOL) --add-modules $(JRE_COMPACT2_MODULES_LIST) \
+	$(call ExecuteWithLog, $(SUPPORT_OUTPUTDIR)/images/jre_compact2, \
-	    $(JLINK_JRE_EXTRA_OPTS) \
+	    $(JLINK_TOOL) --add-modules $(JRE_COMPACT2_MODULES_LIST) \
-	    --output $(JRE_COMPACT2_IMAGE_DIR)
+	        $(JLINK_JRE_EXTRA_OPTS) \
+	        --output $(JRE_COMPACT2_IMAGE_DIR) \
+	)
 	$(TOUCH) $@
 
 $(JRE_COMPACT3_IMAGE_DIR)/$(JIMAGE_TARGET_FILE): $(JMODS) \
     $(call DependOnVariable, JRE_COMPACT3_MODULES_LIST) $(BASE_RELEASE_FILE)
 	$(ECHO) Creating jre compact3 jimage
 	$(RM) -r $(JRE_COMPACT3_IMAGE_DIR)
-	$(JLINK_TOOL) --add-modules $(JRE_COMPACT3_MODULES_LIST) \
+	$(call ExecuteWithLog, $(SUPPORT_OUTPUTDIR)/images/jre_compact3, \
-	    $(JLINK_JRE_EXTRA_OPTS) \
+	    $(JLINK_TOOL) --add-modules $(JRE_COMPACT3_MODULES_LIST) \
-	    --output $(JRE_COMPACT3_IMAGE_DIR)
+	        $(JLINK_JRE_EXTRA_OPTS) \
+	        --output $(JRE_COMPACT3_IMAGE_DIR) \
+	)
 	$(TOUCH) $@
 
 TOOL_JRE_TARGETS := $(JRE_IMAGE_DIR)/$(JIMAGE_TARGET_FILE)

make/autoconf/basics.m4

@@ -258,16 +258,12 @@ AC_DEFUN([BASIC_REMOVE_SYMBOLIC_LINKS],
     if test "x$READLINK_TESTED" != yes; then
       # On MacOSX there is a readlink tool with a different
       # purpose than the GNU readlink tool. Check the found readlink.
-      ISGNU=`$READLINK --version 2>&1 | $GREP GNU`
+      READLINK_ISGNU=`$READLINK --version 2>&1 | $GREP GNU`
-      if test "x$ISGNU" = x; then
+      # If READLINK_ISGNU is empty, then it's a non-GNU readlink. Don't use it.
-        # A readlink that we do not know how to use.
+      READLINK_TESTED=yes
-        # Are there other non-GNU readlinks out there?
-        READLINK_TESTED=yes
-        READLINK=
-      fi
     fi
 
-    if test "x$READLINK" != x; then
+    if test "x$READLINK" != x && "x$READLINK_ISGNU" != x; then
       $1=`$READLINK -f [$]$1`
     else
       # Save the current directory for restoring afterwards
@@ -494,6 +490,7 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS],
   BASIC_REQUIRE_PROGS(MV, mv)
   BASIC_REQUIRE_PROGS(NAWK, [nawk gawk awk])
   BASIC_REQUIRE_PROGS(PRINTF, printf)
+  BASIC_REQUIRE_PROGS(READLINK, [greadlink readlink])
   BASIC_REQUIRE_PROGS(RM, rm)
   BASIC_REQUIRE_PROGS(RMDIR, rmdir)
   BASIC_REQUIRE_PROGS(SH, sh)
@@ -525,7 +522,6 @@ AC_DEFUN_ONCE([BASIC_SETUP_FUNDAMENTAL_TOOLS],
 
   # These are not required on all platforms
   BASIC_PATH_PROGS(CYGPATH, cygpath)
-  BASIC_PATH_PROGS(READLINK, [greadlink readlink])
   BASIC_PATH_PROGS(DF, df)
   BASIC_PATH_PROGS(CPIO, [cpio bsdcpio])
   BASIC_PATH_PROGS(NICE, nice)

make/autoconf/spec.gmk.in

@@ -685,6 +685,7 @@ NICE:=@NICE@
 PANDOC:=@FIXPATH@ @PANDOC@
 PATCH:=@PATCH@
 PRINTF:=@PRINTF@
+READLINK:=@READLINK@
 RM:=@RM@
 RMDIR:=@RMDIR@
 SED:=@SED@

make/common/MakeBase.gmk

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2011, 2017, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -545,8 +545,18 @@ else ifeq ($(OPENJDK_TARGET_OS),macosx)
   define install-file
 	$(call MakeTargetDir)
 	$(RM) '$(call DecodeSpace, $@)'
-	$(CP) -fRP '$(call DecodeSpace, $<)' '$(call DecodeSpace, $@)'
+        # Work around a weirdness with cp on Macosx. When copying a symlink, if
-	if [ -n "`$(XATTR) -ls '$(call DecodeSpace, $@)'`" ]; then $(XATTR) -cs '$(call DecodeSpace, $@)'; fi
+        # the target of the link is write protected (e.g. 444), cp will add
+        # write permission for the user on the target file (644). Avoid this by
+        # using ln to create a new link instead.
+	if [ -h '$(call DecodeSpace, $<)' ]; then \
+	  $(LN) -s "`$(READLINK) '$(call DecodeSpace, $<)'`" '$(call DecodeSpace, $@)'; \
+	else \
+	  $(CP) -fRP '$(call DecodeSpace, $<)' '$(call DecodeSpace, $@)'; \
+	fi
+	if [ -n "`$(XATTR) -ls '$(call DecodeSpace, $@)'`" ]; then \
+	  $(XATTR) -cs '$(call DecodeSpace, $@)'; \
+	fi
   endef
 else
   define install-file
@@ -1000,6 +1010,7 @@ endif
 # Param 2 - The command to run
 ExecuteWithLog = \
   $(call LogCmdlines, Exececuting: [$(strip $2)]) \
+  $(call MakeDir, $(dir $(strip $1))) \
   $(call WriteFile, $2, $(strip $1).cmdline) \
   ( $(strip $2) > >($(TEE) $(strip $1).log) 2> >($(TEE) $(strip $1).log >&2) || \
       ( exitcode=$(DOLLAR)? && \

make/scripts/compare.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -94,7 +94,7 @@ diff_text() {
             $SED -e '/[<>] \* from.*\.idl/d' \
                  -e '/[<>] .*[0-9]\{4\}_[0-9]\{2\}_[0-9]\{2\}_[0-9]\{2\}_[0-9]\{2\}-b[0-9]\{2\}.*/d' \
                  -e '/[<>] .*[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}-[0-9]\{6\}.*/d' \
-                 -e '/[<>] \*.*[0-9]\{4\} [0-9][0-9]*:[0-9]\{2\}:[0-9]\{2\}.*/d' \
+                 -e '/[<>] \*.*[0-9]\{4\} \(at \)*[0-9][0-9]*:[0-9]\{2\}:[0-9]\{2\}.*/d' \
                  -e '/\/\/ Generated from input file.*/d' \
                  -e '/\/\/ This file was generated AUTOMATICALLY from a template file.*/d' \
                  -e '/\/\/ java GenerateCharacter.*/d')
@@ -102,9 +102,12 @@ diff_text() {
     # Ignore date strings in class files.
     # Anonymous lambda classes get randomly assigned counters in their names.
     if test "x$SUFFIX" = "xclass"; then
-        if [ "$NAME" = "SystemModules.class" ]; then
+        if [ "$NAME" = "SystemModules\$all.class" ] \
-            # The SystemModules.class is not comparable. The way it is generated is
+           || [ "$NAME" = "SystemModules\$default.class" ]; then
-            # too random. It can even be of different size for no apparent reason.
+            # The SystemModules\$*.classes are not comparable as they contain the
+            # module hashes which would require a whole other level of
+            # reproducible builds to get reproducible. There is also random
+            # order of map initialization.
             TMP=""
         elif [ "$NAME" = "module-info.class" ]; then
             # The module-info.class have several issues with random ordering of
@@ -654,7 +657,6 @@ compare_bin_file() {
         # pdb files.
         PDB_DIRS="$(ls -d \
             {$OTHER,$THIS}/support/modules_{cmds,libs}/{*,*/*} \
-            {$OTHER,$THIS}/support/demos/image/jvmti/*/lib \
             {$OTHER,$THIS}/support/native/java.base/java_objs \
             )"
         export _NT_SYMBOL_PATH="$(echo $PDB_DIRS | tr ' ' ';')"
@@ -1001,6 +1003,12 @@ compare_all_libs() {
         -o -name '*.dll' -o -name '*.obj' -o -name '*.o' -o -name '*.a' \
         -o -name '*.cpl' \) | $SORT | $FILTER)
 
+    # On macos, filter out the dSYM debug symbols files as they are also
+    # named *.dylib.
+    if [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
+        LIBS=$(echo "$LIBS" | $GREP -v '\.dSYM/')
+    fi
+
     if [ -n "$LIBS" ]; then
         echo Libraries...
         print_binary_diff_header
@@ -1041,7 +1049,7 @@ compare_all_execs() {
             -o -name '*.jfc' -o -name '*.dat'  -o -name 'release' -o -name '*.dir'\
             -o -name '*.sym' -o -name '*.idl' -o -name '*.h' -o -name '*.access' \
             -o -name '*.template' -o -name '*.policy' -o -name '*.security' \
-            -o -name 'COPYRIGHT' -o -name '*.1' \
+            -o -name 'COPYRIGHT' -o -name '*.1' -o -name '*.debuginfo' \
             -o -name 'classlist' \) | $SORT | $FILTER)
     fi
 
@@ -1219,7 +1227,7 @@ if [ "$STRIP_ALL" = "true" ] && [ -z "$STRIP" ]; then
   STRIP_ALL=false
 fi
 
-COMPARE_ROOT=/tmp/cimages.$USER
+COMPARE_ROOT=$OUTPUTDIR/compare-support
 if [ "$CLEAN_OUTPUT" = "true" ]; then
     echo Cleaning old output in $COMPARE_ROOT.
     $RM -rf $COMPARE_ROOT

make/scripts/compare_exceptions.sh.incl

@@ -428,8 +428,13 @@ if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
       ./bin/xjc.exe
       "
 
+  SORT_SYMBOLS="
+      ./bin/fontmanager.dll
+      "
+
   ACCEPTED_DIS_DIFF="
       ./bin/jabswitch.exe
+      ./bin/fontmanager.dll
       "
 
   if [ "$OPENJDK_TARGET_CPU" = "x86" ]; then