8169966: Larger AWT menus

Reviewed-by: azvegint, prr, rhalade, mschoene

src/java.desktop/windows/native/libawt/windows/CmdIDList.cpp

@@ -61,21 +61,29 @@ INLINE void AwtCmdIDList::BuildFreeList(UINT first_index)
     m_first_free = first_index; // head of the free list
 }
 
+
+jboolean AwtCmdIDList::isFreeIDAvailable() {
+    CriticalSection::Lock l(m_lock);
+
+    if (m_first_free == -1) {   // out of free ids
+        if (m_capacity == ARRAY_MAXIMUM_SIZE) {
+            return JNI_FALSE;
+        }
+    }
+    return JNI_TRUE;
+}
+
 // Assign an id to the object.  Recycle the first free entry from the
 // head of the free list or allocate more memory for a new free list.
 UINT AwtCmdIDList::Add(AwtObject* obj)
 {
     CriticalSection::Lock l(m_lock);
+    if (!isFreeIDAvailable()) {
+        throw std::bad_alloc(); // fatal error
+    }
 
     if (m_first_free == -1) {   // out of free ids
-        if (m_capacity == ARRAY_MAXIMUM_SIZE) {
+        // snarf a bigger arena
-            // Really bad - out of ids.  Since we hardly can have *so*
-            // many items simultaneously in existence, we have an id
-            // leak somewhere.
-            DASSERT(FALSE);
-            return 0;
-        }
-        else {                  // snarf a bigger arena
         UINT old_capacity = m_capacity; // will be the first free entry
         m_capacity += ARRAY_SIZE_INCREMENT;
         if (m_capacity > ARRAY_MAXIMUM_SIZE)
@@ -84,7 +92,6 @@ UINT AwtCmdIDList::Add(AwtObject* obj)
                                     m_capacity, sizeof(CmdIDEntry*));
         BuildFreeList(old_capacity);
     }
-    }
 
     DASSERT(m_first_free != -1);
     UINT newid = m_first_free;  // use the entry from the head of the list

src/java.desktop/windows/native/libawt/windows/CmdIDList.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 1999, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,7 @@ public:
     UINT Add(AwtObject* obj);
     AwtObject* Lookup(UINT id);
     void Remove(UINT id);
+    jboolean isFreeIDAvailable();
 
     CriticalSection    m_lock;
 

src/java.desktop/windows/native/libawt/windows/awt_MenuItem.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -217,6 +217,10 @@ AwtMenuItem* AwtMenuItem::Create(jobject peer, jobject menuPeer)
         if (env->EnsureLocalCapacity(1) < 0) {
             return NULL;
         }
+        if (!AwtToolkit::GetInstance().isFreeIDAvailable()) {
+            return NULL;
+        }
+
         JNI_CHECK_NULL_RETURN_NULL(menuPeer, "peer");
 
         /* target is a java.awt.MenuItem  */

src/java.desktop/windows/native/libawt/windows/awt_Toolkit.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1869,6 +1869,11 @@ void AwtToolkit::SyncCall(void (*ftn)(void)) {
     }
 }
 
+jboolean AwtToolkit::isFreeIDAvailable()
+{
+    return m_cmdIDs->isFreeIDAvailable();
+}
+
 UINT AwtToolkit::CreateCmdID(AwtObject* object)
 {
     return m_cmdIDs->Add(object);

src/java.desktop/windows/native/libawt/windows/awt_Toolkit.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -373,6 +373,8 @@ public:
     BOOL PreProcessMouseMsg(class AwtComponent* p, MSG& msg);
     BOOL PreProcessKeyMsg(class AwtComponent* p, MSG& msg);
 
+    /* Checks that an free ID exists. */
+    jboolean isFreeIDAvailable();
     /* Create an ID which maps to an AwtObject pointer, such as a menu. */
     UINT CreateCmdID(AwtObject* object);