8253899: Make IsClassUnloadingEnabled signature match specification

Reviewed-by: sspitsyn, dholmes

src/hotspot/share/prims/jvmti.xml

@@ -9949,9 +9949,11 @@ myInit() {
           there is a <code>jint</code> parameter, the event handler should be
           declared:
 <example>
-    void JNICALL myHandler(jvmtiEnv* jvmti_env, jint myInt, ...)
+    void JNICALL myHandler(jvmtiEnv* jvmti_env, ...)
 </example>
           Note the terminal "<code>...</code>" which indicates varargs.
+          The <code>jint</code> argument inside <code>myHandler</code> needs to be extracted using
+          the <code>va_*</code> syntax of the C programming language.
         </description>
         <parameters>
           <param id="jvmti_env">

src/hotspot/share/prims/jvmtiExtensions.cpp

@@ -34,7 +34,14 @@ GrowableArray<jvmtiExtensionEventInfo*>* JvmtiExtensions::_ext_events;
 
 
 // extension function
-static jvmtiError JNICALL IsClassUnloadingEnabled(const jvmtiEnv* env, jboolean* enabled, ...) {
+static jvmtiError JNICALL IsClassUnloadingEnabled(const jvmtiEnv* env, ...) {
+  jboolean* enabled = NULL;
+  va_list ap;
+
+  va_start(ap, env);
+  enabled = va_arg(ap, jboolean *);
+  va_end(ap);
+
   if (enabled == NULL) {
     return JVMTI_ERROR_NULL_POINTER;
   }

test/hotspot/jtreg/vmTestbase/nsk/jvmti/scenarios/extension/EX03/ex03t001/ex03t001.cpp

@@ -41,7 +41,15 @@ static jrawMonitorID eventMon;
 /* ============================================================================= */
 
 static void JNICALL
-ClassUnload(jvmtiEnv* jvmti_env, JNIEnv* jni_env, const char* name, ...) {
+ClassUnload(jvmtiEnv* jvmti_env, ...) {
+    JNIEnv *jni_env = NULL;
+    va_list ap;
+
+    va_start(ap, jvmti_env);
+    jni_env = va_arg(ap, JNIEnv *);
+    const char * name = va_arg(ap, const char *);
+    va_end(ap);
+
     // The name argument should never be null
     if (name == NULL) {
         nsk_jvmti_setFailStatus();