8317959: Check return values of malloc in native java.base coding

Reviewed-by: alanb, bpb

src/java.base/aix/native/libnio/MappedMemoryUtils.c

@@ -158,6 +158,11 @@ static void check_aix_einval(JNIEnv* env, void* end_address)
     FILE* proc_file;
     {
         char* fname = (char*) malloc(sizeof(char) * PFNAME_LEN);
+        if (fname == NULL) {
+            JNU_ThrowOutOfMemoryError(env, NULL);
+            return;
+        }
+
         pid_t the_pid = getpid();
         jio_snprintf(fname, PFNAME_LEN, "/proc/%d/map", the_pid);
         proc_file = fopen(fname, "r");
@@ -170,6 +175,11 @@ static void check_aix_einval(JNIEnv* env, void* end_address)
     }
     {
         prmap_t* map_entry = (prmap_t*) malloc(sizeof(prmap_t));
+        if (map_entry == NULL) {
+            JNU_ThrowOutOfMemoryError(env, NULL);
+            fclose(proc_file);
+            return;
+        }
         check_proc_map_array(env, proc_file, map_entry, end_address);
         free(map_entry);
     }

src/java.base/unix/native/libjava/java_props_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -238,6 +238,11 @@ static int ParseLocale(JNIEnv* env, int cat, char ** std_language, char ** std_s
         *std_language = "en";
         if (language != NULL && mapLookup(language_names, language, std_language) == 0) {
             *std_language = malloc(strlen(language)+1);
+            if (*std_language == NULL) {
+                free(encoding_variant);
+                JNU_ThrowOutOfMemoryError(env, NULL);
+                return 0;
+            }
             strcpy(*std_language, language);
         }
     }
@@ -246,6 +251,11 @@ static int ParseLocale(JNIEnv* env, int cat, char ** std_language, char ** std_s
     if (std_country != NULL && country != NULL) {
         if (mapLookup(country_names, country, std_country) == 0) {
             *std_country = malloc(strlen(country)+1);
+            if (*std_country == NULL) {
+                free(encoding_variant);
+                JNU_ThrowOutOfMemoryError(env, NULL);
+                return 0;
+            }
             strcpy(*std_country, country);
         }
     }

src/java.base/windows/native/libjli/cmdtoargs.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -324,6 +324,10 @@ public:
     bool check() {
         // "pgmname" rest of cmdline ie. pgmname + 2 double quotes + space + cmdline from windows
         char* cptr = (char*) malloc(strlen(argv[0]) + sizeof(char) * 3 + strlen(cmdline) + 1);
+        if (cptr == NULL) {
+            printf("*** cannot allocate memory\n");
+            doabort();
+        }
         _snprintf(cptr, MAX_PATH, "\"%s\" %s", argv[0], cmdline);
         JLI_CmdToArgs(cptr);
         free(cptr);

src/java.base/windows/native/libnio/ch/UnixDomainSockets.c

@@ -118,6 +118,9 @@ Java_sun_nio_ch_UnixDomainSockets_init(JNIEnv *env, jclass cl)
     if (result == SOCKET_ERROR) {
         if (GetLastError() == WSAENOBUFS) {
             infoPtr = (LPWSAPROTOCOL_INFOW)malloc(len);
+            if (infoPtr == NULL) {
+                return JNI_FALSE;
+            }
             result = WSAEnumProtocolsW(0, infoPtr, &len);
             if (result == SOCKET_ERROR) {
                 free(infoPtr);