8230480: check malloc/calloc results in java.desktop

Reviewed-by: rriggs

src/java.desktop/macosx/native/libjsound/PLATFORM_API_MacOSX_Ports.cpp

@@ -414,11 +414,18 @@ static void* CreatePortControl(PortMixer *mixer, PortControlCreator *creator, Po
                                AudioControl **audioControls, int offset, int len) {
     void *jControl = NULL;
     PortControl *control = (PortControl *)calloc(1, sizeof(PortControl));
+    if (control == NULL) {
+        return NULL;
+    }
     float precision = 0.01;
 
     control->type = type;
     control->controlCount = len;
     control->audioControls = (AudioControl **)malloc(len * sizeof(AudioControl *));
+    if (control->audioControls == NULL) {
+        free(control);
+        return NULL;
+    }
     memcpy(control->audioControls, audioControls + offset, len * sizeof(AudioControl *));
 
     switch (control->type) {
@@ -482,6 +489,9 @@ void PORT_GetControls(void* id, INT32 portIndex, PortControlCreator* creator) {
                 OS_ERROR1(err, "PORT_GetControls (portIndex = %d) get OwnedObject values", portIndex);
             } else {
                 mixer->deviceControls = (AudioControl *)calloc(mixer->deviceControlCount, sizeof(AudioControl));
+                if (mixer->deviceControls == NULL) {
+                    return;
+                }
 
                 for (int i = 0; i < mixer->deviceControlCount; i++) {
                     AudioControl *control = &mixer->deviceControls[i];
@@ -615,10 +625,16 @@ void PORT_GetControls(void* id, INT32 portIndex, PortControlCreator* creator) {
                 if (err == noErr) {
                     CFIndex length = CFStringGetLength(cfname) + 1;
                     channelName = (char *)malloc(length);
+                    if (channelName == NULL) {
+                        return;
+                    }
                     CFStringGetCString(cfname, channelName, length, kCFStringEncodingUTF8);
                     CFRelease(cfname);
                 } else {
                     channelName = (char *)malloc(16);
+                    if (channelName == NULL) {
+                        return;
+                    }
                     sprintf(channelName, "Ch %d", ch);
                 }
 

src/java.desktop/unix/native/common/awt/X11Color.c

@@ -402,6 +402,9 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
     pVI = &awt_data->awt_visInfo;
     awt_data->awt_num_colors = awt_data->awt_visInfo.colormap_size;
     awt_data->awtImage = (awtImageData *) calloc (1, sizeof (awtImageData));
+    if (awt_data->awtImage == NULL) {
+        return 0;
+    }
 
     pPFV = XListPixmapFormats(dpy, &numpfv);
     if (pPFV) {
@@ -572,12 +575,17 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
     }
 
     if (awt_data->awt_num_colors > paletteSize) {
-        free (awt_data->awtImage);
+        free(awt_data->awtImage);
         return 0;
     }
 
     /* Allocate ColorData structure */
     awt_data->color_data = ZALLOC (_ColorData);
+    if (awt_data->color_data == NULL) {
+        free(awt_data->awtImage);
+        return 0;
+    }
+
     awt_data->color_data->screendata = 1; /* This ColorData struct corresponds
                                              to some AWT screen/visual, so when
                                              any IndexColorModel using this
@@ -594,6 +602,11 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
 
     awt_data->color_data->awt_Colors =
         (ColorEntry *)calloc(paletteSize, sizeof (ColorEntry));
+    if (awt_data->color_data->awt_Colors == NULL) {
+        free(awt_data->awtImage);
+        free(awt_data->color_data);
+        return 0;
+    }
 
     XQueryColors(dpy, cm, cols, awt_data->awt_num_colors);
     for (i = 0; i < awt_data->awt_num_colors; i++) {
@@ -667,6 +680,11 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
 
         awt_data->color_data->img_grays =
             (unsigned char *)calloc(256, sizeof(unsigned char));
+        if ( awt_data->color_data->img_grays == NULL) {
+            free(awt_data->awtImage);
+            free(awt_data->color_data);
+            return 0;
+        }
         for (g = 0; g < 256; g++) {
             int mindist, besti;
             int d;
@@ -810,6 +828,11 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
     awt_data->color_data->img_clr_tbl =
         (unsigned char *)calloc(LOOKUPSIZE * LOOKUPSIZE * LOOKUPSIZE,
                                 sizeof(unsigned char));
+    if (awt_data->color_data->img_clr_tbl == NULL) {
+        free(awt_data->awtImage);
+        free(awt_data->color_data);
+        return 0;
+    }
     img_makePalette(cmapsize, k, LOOKUPSIZE, 50, 250,
                     allocatedColorsNum, TRUE, reds, greens, blues,
                     awt_data->color_data->img_clr_tbl);
@@ -858,6 +881,12 @@ awt_allocate_colors(AwtGraphicsConfigDataPtr awt_data)
     awt_data->color_data->awt_icmLUT2Colors =
         (unsigned char *)calloc(paletteSize, sizeof (unsigned char));
     awt_data->color_data->awt_icmLUT = (int *)calloc(paletteSize, sizeof(int));
+    if (awt_data->color_data->awt_icmLUT2Colors == NULL || awt_data->color_data->awt_icmLUT == NULL) {
+        free(awt_data->awtImage);
+        free(awt_data->color_data);
+        return 0;
+    }
+
     for (i=0; i < paletteSize; i++) {
         /* Keep the mapping between this lut and the actual cmap */
         awt_data->color_data->awt_icmLUT2Colors

src/java.desktop/unix/native/common/awt/fontpath.c

@@ -341,6 +341,9 @@ static char **getX11FontPath ()
      * cost us a little wasted effort upstream.
      */
     fontdirs = (char**)calloc(nPaths+1, sizeof(char*));
+    if (fontdirs == NULL) {
+        return NULL;
+    }
     pos = 0;
     for (i=0; i < nPaths; i++) {
         if (x11Path[i][0] != '/') {
@@ -420,6 +423,9 @@ static char* mergePaths(char **p1, char **p2, char **p3, jboolean noType1) {
     }
     totalLen = len1+len2+len3;
     fontdirs = (char**)calloc(totalLen, sizeof(char*));
+    if (fontdirs == NULL) {
+        return NULL;
+    }
 
     for (i=0; i < len1; i++) {
         if (noType1 && strstr(p1[i], "Type1") != NULL) {
@@ -816,6 +822,10 @@ static char **getFontConfigLocations() {
         fontdirs = NULL;
     } else {
         fontdirs = (char**)calloc(fontSet->nfont+1, sizeof(char*));
+        if (fontdirs == NULL) {
+            (*FcFontSetDestroy)(fontSet);
+            goto cleanup;
+        }
         for (f=0; f < fontSet->nfont; f++) {
             FcChar8 *file;
             FcChar8 *dir;
@@ -840,6 +850,7 @@ static char **getFontConfigLocations() {
         (*FcFontSetDestroy)(fontSet);
     }
 
+cleanup:
     /* Free memory and close the ".so" */
     (*FcPatternDestroy)(pattern);
     closeFontConfig(libfontconfig, JNI_TRUE);

src/java.desktop/unix/native/libawt_xawt/awt/gtk_interface.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -66,6 +66,9 @@ static GtkLib** get_libs_order(GtkVersion version) {
     if (!n_libs) {
         n_libs = sizeof(gtk_libs) / sizeof(GtkLib);
         load_order = calloc(n_libs + 1, sizeof(GtkLib *));
+        if (load_order == NULL) {
+          return NULL;
+        }
     }
     int i, first = 0;
     for (i = 0; i < n_libs; i++) {
@@ -85,6 +88,7 @@ static GtkLib** get_libs_order(GtkVersion version) {
 
 static GtkLib* get_loaded() {
     GtkLib** libs = get_libs_order(GTK_ANY);
+    if (libs == NULL) return NULL;
     while(!gtk && *libs) {
         GtkLib* lib = *libs++;
         if (lib->check(lib->vname, /* load = */FALSE)) {
@@ -111,7 +115,7 @@ gboolean gtk_load(JNIEnv *env, GtkVersion version, gboolean verbose) {
             }
         } else {
             GtkLib** libs = get_libs_order(version);
-            while (!gtk && *libs) {
+            while (!gtk && libs && *libs) {
                 lib = *libs++;
                 if (version == GTK_ANY || lib->version == version) {
                     if (verbose) {
@@ -141,6 +145,7 @@ gboolean gtk_load(JNIEnv *env, GtkVersion version, gboolean verbose) {
 
 static gboolean check_version(GtkVersion version) {
     GtkLib** libs = get_libs_order(version);
+    if (libs == NULL) return FALSE;
     while (*libs) {
         GtkLib* lib = *libs++;
         if (lib->check(lib->vname, /* load = */TRUE)) {