8212779: ADL Parser does not check allocation return values in all cases

Made to fail gracefully in case of malloc failure.

Reviewed-by: kvn

src/hotspot/share/adlc/adlparse.cpp

@@ -210,7 +210,7 @@ void ADLParser::instr_parse(void) {
             return;
           }
           assert(match_rules_cnt < 100," too many match rule clones");
-          char* buf = (char*) malloc(strlen(instr->_ident) + 4);
+          char* buf = (char*) AllocateHeap(strlen(instr->_ident) + 4);
           sprintf(buf, "%s_%d", instr->_ident, match_rules_cnt++);
           rule->_result = buf;
           // Check for commutative operations with tree operands.
@@ -2858,7 +2858,7 @@ void ADLParser::ins_encode_parse_block(InstructForm& inst) {
   // Create a new encoding name based on the name of the instruction
   // definition, which should be unique.
   const char* prefix = "__ins_encode_";
-  char* ec_name = (char*) malloc(strlen(inst._ident) + strlen(prefix) + 1);
+  char* ec_name = (char*) AllocateHeap(strlen(inst._ident) + strlen(prefix) + 1);
   sprintf(ec_name, "%s%s", prefix, inst._ident);
 
   assert(_AD._encode->encClass(ec_name) == NULL, "shouldn't already exist");
@@ -3328,7 +3328,7 @@ void ADLParser::constant_parse(InstructForm& inst) {
   // Create a new encoding name based on the name of the instruction
   // definition, which should be unique.
   const char* prefix = "__constant_";
-  char* ec_name = (char*) malloc(strlen(inst._ident) + strlen(prefix) + 1);
+  char* ec_name = (char*) AllocateHeap(strlen(inst._ident) + strlen(prefix) + 1);
   sprintf(ec_name, "%s%s", prefix, inst._ident);
 
   assert(_AD._encode->encClass(ec_name) == NULL, "shouldn't already exist");
@@ -4460,7 +4460,7 @@ char* ADLParser::find_cpp_block(const char* description) {
     if (_AD._adlocation_debug) {
       char* location = get_line_string(line);
       char* end_loc  = end_line_marker();
-      char* result = (char *)malloc(strlen(location) + strlen(cppBlock) + strlen(end_loc) + 1);
+      char* result = (char *)AllocateHeap(strlen(location) + strlen(cppBlock) + strlen(end_loc) + 1);
       strcpy(result, location);
       strcat(result, cppBlock);
       strcat(result, end_loc);
@@ -4549,7 +4549,7 @@ char *ADLParser::get_paren_expr(const char *description, bool include_location)
     // Prepend location descriptor, for debugging.
     char* location = get_line_string(line);
     char* end_loc  = end_line_marker();
-    char* result = (char *)malloc(strlen(location) + strlen(token2) + strlen(end_loc) + 1);
+    char* result = (char *)AllocateHeap(strlen(location) + strlen(token2) + strlen(end_loc) + 1);
     strcpy(result, location);
     strcat(result, token2);
     strcat(result, end_loc);
@@ -4647,7 +4647,7 @@ char *ADLParser::get_ident_or_literal_constant(const char* description) {
     // Grab a constant expression.
     param = get_paren_expr(description);
     if (param[0] != '(') {
-      char* buf = (char*) malloc(strlen(param) + 3);
+      char* buf = (char*) AllocateHeap(strlen(param) + 3);
       sprintf(buf, "(%s)", param);
       param = buf;
     }
@@ -5255,7 +5255,7 @@ void ADLParser::next_line() {
 char* ADLParser::get_line_string(int linenum) {
   const char* file = _AD._ADL_file._name;
   int         line = linenum ? linenum : this->linenum();
-  char* location = (char *)malloc(strlen(file) + 100);
+  char* location = (char *)AllocateHeap(strlen(file) + 100);
   sprintf(location, "\n#line %d \"%s\"\n", line, file);
   return location;
 }

src/hotspot/share/adlc/arena.cpp

@@ -24,6 +24,16 @@
 
 #include "adlc.hpp"
 
+void* AllocateHeap(size_t size) {
+  unsigned char* ptr = (unsigned char*) malloc(size);
+  if (ptr == NULL && size != 0) {
+    fprintf(stderr, "Error: Out of memory in ADLC\n"); // logging can cause crash!
+    fflush(stderr);
+    exit(1);
+  }
+  return ptr;
+}
+
 void* Chunk::operator new(size_t requested_size, size_t length) throw() {
   return CHeapObj::operator new(requested_size + length);
 }
@@ -164,7 +174,7 @@ bool Arena::contains( const void *ptr ) const {
 // CHeapObj
 
 void* CHeapObj::operator new(size_t size) throw() {
-  return (void *) malloc(size);
+  return (void *) AllocateHeap(size);
 }
 
 void CHeapObj::operator delete(void* p){

src/hotspot/share/adlc/arena.hpp

@@ -25,6 +25,8 @@
 #ifndef SHARE_VM_ADLC_ARENA_HPP
 #define SHARE_VM_ADLC_ARENA_HPP
 
+void* AllocateHeap(size_t size);
+
 // All classes in adlc may be derived
 // from one of the following allocation classes:
 //
@@ -42,7 +44,6 @@ class CHeapObj {
   void* new_array(size_t size);
 };
 
-
 // Base class for classes that constitute name spaces.
 
 class AllStatic {

src/hotspot/share/adlc/forms.cpp

@@ -40,7 +40,7 @@ const char  *NameList::_signal3  = "$$SIGNAL3$$";
 
 // Constructor and Destructor
 NameList::NameList() : _cur(0), _max(4), _iter(0), _justReset(true) {
-  _names = (const char**)malloc(_max*sizeof(char*));
+  _names = (const char**) AllocateHeap(_max*sizeof(char*));
 }
 NameList::~NameList() {
   // The following free is a double-free, and crashes the program:

src/hotspot/share/adlc/formssel.cpp

@@ -1354,7 +1354,7 @@ void InstructForm::set_unique_opnds() {
     // component back to an index and any DEF always goes at 0 so the
     // length of the array has to be the number of components + 1.
     _uniq_idx_length = _components.count() + 1;
-    uniq_idx = (uint*) malloc(sizeof(uint) * _uniq_idx_length);
+    uniq_idx = (uint*) AllocateHeap(sizeof(uint) * _uniq_idx_length);
     for (i = 0; i < _uniq_idx_length; i++) {
       uniq_idx[i] = i;
     }
@@ -3449,7 +3449,7 @@ void MatchNode::build_internalop( ) {
   rstr = (_rChild) ? ((_rChild->_internalop) ?
                        _rChild->_internalop : _rChild->_opType) : "";
   len += (int)strlen(lstr) + (int)strlen(rstr);
-  subtree = (char *)malloc(len);
+  subtree = (char *)AllocateHeap(len);
   sprintf(subtree,"_%s_%s_%s", _opType, lstr, rstr);
   // Hash the subtree string in _internalOps; if a name exists, use it
   iop = (char *)_AD._internalOps[subtree];
@@ -3863,7 +3863,7 @@ void MatchRule::matchrule_swap_commutative_op(const char* instr_ident, int count
   MatchRule* clone = new MatchRule(_AD, this);
   // Swap operands of commutative operation
   ((MatchNode*)clone)->swap_commutative_op(true, count);
-  char* buf = (char*) malloc(strlen(instr_ident) + 4);
+  char* buf = (char*) AllocateHeap(strlen(instr_ident) + 4);
   sprintf(buf, "%s_%d", instr_ident, match_rules_cnt++);
   clone->_result = buf;