8214720: Add pandoc filter to improve html man page output

Co-authored-by: Jonathan Gibbons <jonathan.gibbons@oracle.com>
Reviewed-by: erikj

make/CompileToolsJdk.gmk

@@ -91,15 +91,17 @@ TARGETS += $(COMPILE_DEPEND) $(DEPEND_SERVICE_PROVIDER)
 # To be able to call the javascript filter when generating man pages using
 # pandoc, we need to create this executable wrapper script.
 ifneq ($(PANDOC), )
-  # PANDOC_FILTER is duplicated for export in ToolsJdk.gmk.
+  # PANDOC_TROFF_MANPAGE_FILTER is duplicated for export in ToolsJdk.gmk.
-  PANDOC_FILTER := $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-manpage-filter
+  PANDOC_TROFF_MANPAGE_FILTER := \
-  PANDOC_FILTER_SETUP := $(BUILDTOOLS_OUTPUTDIR)/manpages/_pandoc_filter_setup.marker
+      $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-troff-manpage-filter
+  PANDOC_TROFF_MANPAGE_FILTER_SETUP := \
+      $(BUILDTOOLS_OUTPUTDIR)/manpages/_pandoc_troff_manpage_filter_setup.marker
 
   # Create a usable instance of the wrapper script that calls the pandoc filter
   # (which is written in javascript).
-  $(eval $(call SetupTextFileProcessing, CREATE_PANDOC_FILTER, \
+  $(eval $(call SetupTextFileProcessing, CREATE_PANDOC_TROFF_MANPAGE_FILTER, \
-      SOURCE_FILES := $(TOPDIR)/make/scripts/pandoc-manpage-filter.sh.template, \
+      SOURCE_FILES := $(TOPDIR)/make/scripts/pandoc-troff-manpage-filter.sh.template, \
-      OUTPUT_FILE := $(PANDOC_FILTER), \
+      OUTPUT_FILE := $(PANDOC_TROFF_MANPAGE_FILTER), \
       REPLACEMENTS := \
           @@BOOT_JDK@@ => $(BOOT_JDK) ; \
           @@TOPDIR@@ => $(TOPDIR) ; \
@@ -107,11 +109,35 @@ ifneq ($(PANDOC), )
   ))
 
   # Created script must be made executable
-  $(PANDOC_FILTER_SETUP): $(CREATE_PANDOC_FILTER)
+  $(PANDOC_TROFF_MANPAGE_FILTER_SETUP): $(CREATE_PANDOC_TROFF_MANPAGE_FILTER)
-	$(CHMOD) a+rx $(PANDOC_FILTER)
+	$(CHMOD) a+rx $(PANDOC_TROFF_MANPAGE_FILTER)
 	$(TOUCH) $@
 
-  TARGETS += $(PANDOC_FILTER_SETUP)
+  TARGETS += $(PANDOC_TROFF_MANPAGE_FILTER_SETUP)
+
+  # PANDOC_HTML_MANPAGE_FILTER is duplicated for export in ToolsJdk.gmk.
+  PANDOC_HTML_MANPAGE_FILTER := \
+      $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-html-manpage-filter
+  PANDOC_HTML_MANPAGE_FILTER_SETUP := \
+      $(BUILDTOOLS_OUTPUTDIR)/manpages/_pandoc_html_manpage_filter_setup.marker
+
+  # Create a usable instance of the wrapper script that calls the pandoc filter
+  # (which is written in javascript).
+  $(eval $(call SetupTextFileProcessing, CREATE_PANDOC_HTML_MANPAGE_FILTER, \
+      SOURCE_FILES := $(TOPDIR)/make/scripts/pandoc-html-manpage-filter.sh.template, \
+      OUTPUT_FILE := $(PANDOC_HTML_MANPAGE_FILTER), \
+      REPLACEMENTS := \
+          @@BOOT_JDK@@ => $(BOOT_JDK) ; \
+          @@TOPDIR@@ => $(TOPDIR) ; \
+          @@JJS_FLAGS@@ => $(addprefix -J, $(JAVA_FLAGS_SMALL)), \
+  ))
+
+  # Created script must be made executable
+  $(PANDOC_HTML_MANPAGE_FILTER_SETUP): $(CREATE_PANDOC_HTML_MANPAGE_FILTER)
+	$(CHMOD) a+rx $(PANDOC_HTML_MANPAGE_FILTER)
+	$(TOUCH) $@
+
+  TARGETS += $(PANDOC_HTML_MANPAGE_FILTER_SETUP)
 endif
 
 all: $(TARGETS)

make/Docs.gmk

@@ -27,10 +27,10 @@ default: all
 include $(SPEC)
 include MakeBase.gmk
 include Modules.gmk
+include ModuleTools.gmk
 include ProcessMarkdown.gmk
+include ToolsJdk.gmk
 include ZipArchive.gmk
-include $(TOPDIR)/make/ToolsJdk.gmk
-include $(TOPDIR)/make/ModuleTools.gmk
 
 # This is needed to properly setup DOCS_MODULES.
 $(eval $(call ReadImportMetaData))
@@ -556,6 +556,15 @@ ifeq ($(ENABLE_FULL_DOCS), true)
 
   # For all markdown files in $module/share/man directories, convert them to
   # html.
+
+  # Create dynamic man pages from markdown using pandoc. We need
+  # PANDOC_HTML_MANPAGE_FILTER, a wrapper around
+  # PANDOC_HTML_MANPAGE_FILTER_JAVASCRIPT. This is created by buildtools-jdk.
+
+  # We should also depend on the source javascript filter
+  PANDOC_HTML_MANPAGE_FILTER_JAVASCRIPT := \
+      $(TOPDIR)/make/scripts/pandoc-html-manpage-filter.js
+
   $(foreach m, $(ALL_MODULES), \
     $(eval MAN_$m := $(call FindModuleManDirs, $m)) \
     $(foreach d, $(MAN_$m), \
@@ -565,8 +574,11 @@ ifeq ($(ENABLE_FULL_DOCS), true)
             SRC := $d, \
             FILES := $(filter %.md, $(call CacheFind, $d)), \
             DEST := $(DOCS_OUTPUTDIR)/specs/man, \
+            FILTER := $(PANDOC_HTML_MANPAGE_FILTER), \
             CSS := $(GLOBAL_SPECS_DEFAULT_CSS_FILE), \
             REPLACEMENTS := @@VERSION_SHORT@@ => $(VERSION_SHORT), \
+            EXTRA_DEPS := $(PANDOC_HTML_MANPAGE_FILTER) \
+                $(PANDOC_HTML_MANPAGE_FILTER_JAVASCRIPT), \
         )) \
         $(eval JDK_SPECS_TARGETS += $($($m_$d_NAME))) \
       ) \

make/ToolsJdk.gmk

@@ -118,7 +118,8 @@ TOOL_PUBLICSUFFIXLIST = $(JAVA_SMALL) -cp $(BUILDTOOLS_OUTPUTDIR)/jdk_tools_clas
 
 # Executable javascript filter for man page generation using pandoc.
 
-PANDOC_FILTER := $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-manpage-filter
+PANDOC_TROFF_MANPAGE_FILTER := $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-troff-manpage-filter
+PANDOC_HTML_MANPAGE_FILTER := $(BUILDTOOLS_OUTPUTDIR)/manpages/pandoc-html-manpage-filter
 
 ##########################################################################################
 

make/launcher/LauncherCommon.gmk

@@ -207,11 +207,12 @@ ifeq ($(OPENJDK_TARGET_OS_TYPE), unix)
       $(info Warning: pandoc not found. Not generating man pages)
     else
       # Create dynamic man pages from markdown using pandoc. We need
-      # PANDOC_FILTER, a wrapper around PANDOC_FILTER_JAVASCRIPT. This is
+      # PANDOC_TROFF_MANPAGE_FILTER, a wrapper around
-      # created by buildtools-jdk.
+      # PANDOC_TROFF_MANPAGE_FILTER_JAVASCRIPT. This is created by buildtools-jdk.
 
       # We should also depend on the source javascript filter
-      PANDOC_FILTER_JAVASCRIPT := $(TOPDIR)/make/scripts/pandoc-manpage-filter.js
+      PANDOC_TROFF_MANPAGE_FILTER_JAVASCRIPT := \
+          $(TOPDIR)/make/scripts/pandoc-troff-manpage-filter.js
 
       # The norm in man pages is to display code literals as bold, but pandoc
       # "correctly" converts these constructs (encoded in markdown using `...`
@@ -234,10 +235,11 @@ ifeq ($(OPENJDK_TARGET_OS_TYPE), unix)
           DEST := $(SUPPORT_OUTPUTDIR)/modules_man/$(MODULE)/man1, \
           FILES := $(MAN_FILES_MD), \
           FORMAT := man, \
-          FILTER := $(PANDOC_FILTER), \
+          FILTER := $(PANDOC_TROFF_MANPAGE_FILTER), \
           POST_PROCESS := $(MAN_POST_PROCESS), \
           REPLACEMENTS := @@VERSION_SHORT@@ => $(VERSION_SHORT), \
-          EXTRA_DEPS := $(PANDOC_FILTER) $(PANDOC_FILTER_JAVASCRIPT), \
+          EXTRA_DEPS := $(PANDOC_TROFF_MANPAGE_FILTER) \
+              $(PANDOC_TROFF_MANPAGE_FILTER_JAVASCRIPT), \
       ))
 
       TARGETS += $(BUILD_MAN_PAGES)

make/scripts/pandoc-html-manpage-filter.js

@@ -0,0 +1,125 @@
+//
+// Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+//
+// Traverse a tree of pandoc format objects, calling callback on each
+// element, and replacing it if callback returns a new object.
+//
+// Inspired by the walk method in
+// https://github.com/jgm/pandocfilters/blob/master/pandocfilters.py
+//
+function traverse(obj, callback) {
+    if (Array.isArray(obj)) {
+        var processed_array = [];
+        obj.forEach(function(elem) {
+            if (elem === Object(elem) && elem.t) {
+                var replacement = callback(elem.t, elem.c || []);
+                if (!replacement) {
+                    // no replacement object returned, use original
+                    processed_array.push(traverse(elem, callback));
+                } else if (Array.isArray(replacement)) {
+                    // array of objects returned, splice all elements into array
+                    replacement.forEach(function(repl_elem) {
+                        processed_array.push(traverse(repl_elem, callback));
+                    })
+                } else {
+                    // replacement object given, traverse it
+                    processed_array.push(traverse(replacement, callback));
+                }
+            } else {
+                processed_array.push(traverse(elem, callback));
+            }
+        })
+        return processed_array;
+    } else if (obj === Object(obj)) {
+        if (obj.t) {
+            var replacement = callback(obj.t, obj.c || []);
+            if (replacement) {
+                return replacement;
+            }
+        }
+        var processed_obj = {};
+        Object.keys(obj).forEach(function(key) {
+            processed_obj[key] = traverse(obj[key], callback);
+        })
+        return processed_obj;
+    } else {
+        return obj;
+    }
+}
+
+//
+// Helper constructors to create pandoc format objects
+//
+function Space() {
+    return { 't': 'Space' };
+}
+
+function Str(value) {
+    return { 't': 'Str', 'c': value };
+}
+
+function MetaInlines(value) {
+    return { 't': 'MetaInlines', 'c': value };
+}
+
+function change_title(type, value) {
+    if (type === 'MetaInlines') {
+        if (value[0].t === 'Str') {
+            var match = value[0].c.match(/^([A-Z]+)\([0-9]+\)$/);
+            if (match) {
+                return MetaInlines([
+                        Str("The"), Space(),
+			Str(match[1].toLowerCase()),
+			Space(), Str("Command")
+		    ]);
+            }
+        }
+    }
+}
+
+//
+// Main function
+//
+function main() {
+    var input = "";
+    while (line = readLine()) {
+        input = input.concat(line);
+    }
+
+    var json = JSON.parse(input);
+
+    var meta = json.meta;
+    if (meta) {
+        meta.date = undefined;
+        var title = meta.title;
+        if (meta.title) {
+            meta.title = traverse(meta.title, change_title);
+        }
+    }
+
+    print(JSON.stringify(json));
+}
+
+// ... and execute it
+main();

make/scripts/pandoc-html-manpage-filter.sh.template

@@ -25,4 +25,4 @@
 # Simple wrapper script to call Nashorn with the javascript pandoc filter
 
 @@BOOT_JDK@@/bin/jjs @@JJS_FLAGS@@ -scripting \
-    "@@TOPDIR@@/make/scripts/pandoc-manpage-filter.js" 2> /dev/null
+    "@@TOPDIR@@/make/scripts/pandoc-html-manpage-filter.js" 2> /dev/null

make/scripts/pandoc-troff-manpage-filter.sh.template

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+# Simple wrapper script to call Nashorn with the javascript pandoc filter
+
+@@BOOT_JDK@@/bin/jjs @@JJS_FLAGS@@ -scripting \
+    "@@TOPDIR@@/make/scripts/pandoc-troff-manpage-filter.js" 2> /dev/null