8022968: Some codecache allocation failures don't result in invoking the sweeper

Add calls to CompileBroker::handle_full_code_cache() or fail gracefully whenever allocation in the code cache fails.

Reviewed-by: iveresov, vlivanov

hotspot/src/share/vm/code/vtableStubs.cpp

@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "code/vtableStubs.hpp"
+#include "compiler/compileBroker.hpp"
 #include "compiler/disassembler.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/resourceArea.hpp"
@@ -62,6 +63,7 @@ void* VtableStub::operator new(size_t size, int code_size) throw() {
    // If changing the name, update the other file accordingly.
     BufferBlob* blob = BufferBlob::create("vtable chunks", bytes);
     if (blob == NULL) {
+      CompileBroker::handle_full_code_cache();
       return NULL;
     }
     _chunk = blob->content_begin();

hotspot/src/share/vm/interpreter/interpreterRuntime.cpp

@@ -1093,6 +1093,7 @@ IRT_END
 address SignatureHandlerLibrary::set_handler_blob() {
   BufferBlob* handler_blob = BufferBlob::create("native signature handlers", blob_size);
   if (handler_blob == NULL) {
+    CompileBroker::handle_full_code_cache();
     return NULL;
   }
   address handler = handler_blob->code_begin();

hotspot/src/share/vm/opto/compile.cpp

@@ -29,6 +29,7 @@
 #include "classfile/systemDictionary.hpp"
 #include "code/exceptionHandlerTable.hpp"
 #include "code/nmethod.hpp"
+#include "compiler/compileBroker.hpp"
 #include "compiler/compileLog.hpp"
 #include "compiler/disassembler.hpp"
 #include "compiler/oopMap.hpp"
@@ -555,6 +556,7 @@ void Compile::init_scratch_buffer_blob(int const_size) {
     if (scratch_buffer_blob() == NULL) {
       // Let CompilerBroker disable further compilations.
       record_failure("Not enough space for scratch buffer in CodeCache");
+      CompileBroker::handle_full_code_cache();
       return;
     }
   }

hotspot/src/share/vm/opto/output.cpp

@@ -1163,6 +1163,7 @@ CodeBuffer* Compile::init_buffer(uint* blk_starts) {
   // Have we run out of code space?
   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
     C->record_failure("CodeCache is full");
+    CompileBroker::handle_full_code_cache();
     return NULL;
   }
   // Configure the code buffer.
@@ -1487,6 +1488,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
       cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
       if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
         C->record_failure("CodeCache is full");
+        CompileBroker::handle_full_code_cache();
         return;
       }
 
@@ -1643,6 +1645,7 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
   // One last check for failed CodeBuffer::expand:
   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
     C->record_failure("CodeCache is full");
+    CompileBroker::handle_full_code_cache();
     return;
   }
 

hotspot/src/share/vm/runtime/icache.cpp

@@ -34,6 +34,9 @@ void AbstractICache::initialize() {
   ResourceMark rm;
 
   BufferBlob* b = BufferBlob::create("flush_icache_stub", ICache::stub_size);
+  if (b == NULL) {
+    vm_exit_out_of_memory(ICache::stub_size, OOM_MALLOC_ERROR, "CodeCache: no space for flush_icache_stub");
+  }
   CodeBuffer c(b);
 
   ICacheStubGenerator g(&c);