stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8276743: Make openjdk build Zip Archive generation "reproducible"

Browse Source 
Reviewed-by: erikj, ihse
This commit is contained in:
Andrew Leonard 2021-11-12 14:43:54 +00:00
parent 51a5731d6d
commit aeba653034
4 changed files with 261 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
make/Main.gmk
View File

@ -324,7 +324,7 @@ $(eval $(call SetupTarget, vscode-project-ccls, \
# aren't built until after libjava and libjvm are available to link to.
$(eval $(call SetupTarget, demos-jdk, \
MAKEFILE := CompileDemos, \
DEPS := java.base-libs exploded-image, \
DEPS := java.base-libs exploded-image buildtools-jdk, \
))
$(eval $(call SetupTarget, test-image-demos-jdk, \
@ -383,12 +383,12 @@ bootcycle-images:
$(eval $(call SetupTarget, zip-security, \
MAKEFILE := ZipSecurity, \
DEPS := java.base-java java.security.jgss-java java.security.jgss-libs, \
DEPS := buildtools-jdk java.base-java java.security.jgss-java java.security.jgss-libs, \
))
$(eval $(call SetupTarget, zip-source, \
MAKEFILE := ZipSource, \
DEPS := gensrc, \
DEPS := buildtools-jdk gensrc, \
))
$(eval $(call SetupTarget, jrtfs-jar, \
@ -508,13 +508,13 @@ $(eval $(call SetupTarget, docs-jdk-index, \
$(eval $(call SetupTarget, docs-zip, \
MAKEFILE := Docs, \
TARGET := docs-zip, \
DEPS := docs-jdk, \
DEPS := docs-jdk buildtools-jdk, \
))
$(eval $(call SetupTarget, docs-specs-zip, \
MAKEFILE := Docs, \
TARGET := docs-specs-zip, \
DEPS := docs-jdk-specs, \
DEPS := docs-jdk-specs buildtools-jdk, \
))
$(eval $(call SetupTarget, update-build-docs, \

2
make/ToolsJdk.gmk
View File

@ -82,6 +82,8 @@ TOOL_GENERATECACERTS = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_class
TOOL_GENERATEEMOJIDATA = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.generateemojidata.GenerateEmojiData
TOOL_MAKEZIPREPRODUCIBLE = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_classes \
build.tools.makezipreproducible.MakeZipReproducible
# TODO: There are references to the jdwpgen.jar in jdk/make/netbeans/jdwpgen/build.xml
# and nbproject/project.properties in the same dir. Needs to be looked at.

20
make/common/ZipArchive.gmk
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.
# Copyright (c) 2011, 2021, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@ -26,6 +26,9 @@
ifndef _ZIP_ARCHIVE_GMK
_ZIP_ARCHIVE_GMK := 1
# Depends on build tools for MakeZipReproducible
include ../ToolsJdk.gmk
ifeq (,$(_MAKEBASE_GMK))
$(error You must include MakeBase.gmk prior to including ZipArchive.gmk)
endif
@ -134,6 +137,8 @@ define SetupZipArchiveBody
# dir is very small.
# If zip has nothing to do, it returns 12 and would fail the build. Check for 12
# and only fail if it's not.
# For reproducible builds set the zip access & modify times to SOURCE_DATE_EPOCH
# by using a ziptmp folder to generate final zip from using MakeZipReproducible.
$$($1_ZIP) : $$($1_ALL_SRCS) $$($1_EXTRA_DEPS)
$$(call LogWarn, Updating $$($1_NAME))
$$(call MakeTargetDir)
@ -163,7 +168,18 @@ define SetupZipArchiveBody
$$($1_ZIP_EXCLUDES_$$s) \
|| test "$$$$?" = "12" \
))$$(NEWLINE) \
) true \
) true
ifeq ($(ENABLE_REPRODUCIBLE_BUILD), true)
$$(call ExecuteWithLog, \
$$(SUPPORT_OUTPUTDIR)/makezipreproducible/$$(patsubst $$(OUTPUTDIR)/%,%, $$@), \
($(RM) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip && \
$(MKDIR) -p $$(SUPPORT_OUTPUTDIR)/ziptmp/$1 && \
$(TOOL_MAKEZIPREPRODUCIBLE) -f $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip \
-t $(SOURCE_DATE_EPOCH) $$@ && \
$(RM) $$@ && \
$(MV) $$(SUPPORT_OUTPUTDIR)/ziptmp/$1/tmp.zip $$@ \
))
endif
$(TOUCH) $$@
# Add zip to target list

236
make/jdk/src/classes/build/tools/makezipreproducible/MakeZipReproducible.java Normal file
View File

@ -0,0 +1,236 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package build.tools.makezipreproducible;
import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
/**
* Generate a zip file in a "reproducible" manner from the input zip file.
* Standard zip tools rely on OS file list querying whose ordering can vary
* by platform architecture, this class ensures the zip entries are ordered
* and also supports SOURCE_DATE_EPOCH timestamps.
*/
public class MakeZipReproducible {
String input_file = null;
String fname = null;
String zname = "";
long timestamp = -1L;
boolean verbose = false;
// Keep a sorted Set of ZipEntrys to be processed, so that the zip is reproducible
SortedMap<String, ZipEntry> entries = new TreeMap<String, ZipEntry>();
private boolean ok;
public MakeZipReproducible() {
}
public synchronized boolean run(String args[]) {
ok = true;
if (!parseArgs(args)) {
return false;
}
try {
zname = fname.replace(File.separatorChar, '/');
if (zname.startsWith("./")) {
zname = zname.substring(2);
}
if (verbose) System.out.println("Input zip file: " + input_file);
File inFile = new File(input_file);
if (!inFile.exists()) {
error("Input zip file does not exist");
ok = false;
} else {
File zipFile = new File(fname);
// Check archive to create does not exist
if (!zipFile.exists()) {
// Process input ZipEntries
ok = processInputEntries(inFile);
if (ok) {
try (FileOutputStream out = new FileOutputStream(fname)) {
ok = create(inFile, new BufferedOutputStream(out, 4096));
}
} else {
}
} else {
error("Target zip file "+fname+" already exists.");
ok = false;
}
}
} catch (IOException e) {
fatalError(e);
ok = false;
} catch (Error ee) {
ee.printStackTrace();
ok = false;
} catch (Throwable t) {
t.printStackTrace();
ok = false;
}
return ok;
}
boolean parseArgs(String args[]) {
try {
boolean parsingIncludes = false;
boolean parsingExcludes = false;
int count = 0;
while(count < args.length) {
if (args[count].startsWith("-")) {
String flag = args[count].substring(1);
switch (flag.charAt(0)) {
case 'f':
fname = args[++count];
break;
case 't':
// SOURCE_DATE_EPOCH timestamp specified
timestamp = Long.parseLong(args[++count]) * 1000;
break;
case 'v':
verbose = true;
break;
default:
error(String.format("Illegal option -%s", String.valueOf(flag.charAt(0))));
usageError();
return false;
}
} else {
// input zip file
if (input_file != null) {
error("Input zip file already specified");
usageError();
return false;
}
input_file = args[count];
}
count++;
}
} catch (ArrayIndexOutOfBoundsException e) {
usageError();
return false;
} catch (NumberFormatException e) {
usageError();
return false;
}
if (fname == null) {
error("-f <outputArchiveName> must be specified");
usageError();
return false;
}
// If no files specified then default to current directory
if (input_file == null) {
error("No input zip file specified");
usageError();
return false;
}
return true;
}
// Process input zip file and add to sorted entries set
boolean processInputEntries(File inFile) throws IOException {
try (FileInputStream fis = new FileInputStream(inFile);
ZipInputStream zis = new ZipInputStream(fis)) {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
entries.put(entry.getName(), entry);
}
}
return true;
}
// Create new zip from entries
boolean create(File inFile, OutputStream out) throws IOException
{
try (ZipFile zipFile = new ZipFile(inFile);
ZipOutputStream zos = new ZipOutputStream(out)) {
for (Map.Entry<String, ZipEntry> entry : entries.entrySet()) {
ZipEntry zipEntry = entry.getValue();
if (zipEntry.getSize() > 0) {
try (InputStream eis = zipFile.getInputStream(zipEntry)) {
addEntry(zos, zipEntry, eis);
}
} else {
addEntry(zos, zipEntry, null);
}
}
}
return true;
}
// Add Entry and data to Zip
void addEntry(ZipOutputStream zos, ZipEntry entry, InputStream entryInputStream) throws IOException {
if (verbose) {
System.out.println("Adding: "+entry.getName());
}
// Set to specified timestamp if set otherwise leave as original lastModified time
if (timestamp != -1L) {
entry.setTime(timestamp);
}
zos.putNextEntry(entry);
if (entry.getSize() > 0 && entryInputStream != null) {
entryInputStream.transferTo(zos);
}
zos.closeEntry();
}
void usageError() {
error(
"Usage: MakeZipReproducible [-v] [-t <SOURCE_DATE_EPOCH>] -f <output_zip_file> <input_zip_file>\n" +
"Options:\n" +
" -v verbose output\n" +
" -f specify archive file name to create\n" +
" -t specific SOURCE_DATE_EPOCH value to use for timestamps\n" +
" input_zip_file re-written as a reproducible zip output_zip_file.\n");
}
void fatalError(Exception e) {
e.printStackTrace();
}
protected void error(String s) {
System.err.println(s);
}
public static void main(String args[]) {
MakeZipReproducible z = new MakeZipReproducible();
System.exit(z.run(args) ? 0 : 1);
}
}