8321176: [Screencast] make a second attempt on screencast failure

Reviewed-by: azvegint, prr

src/java.desktop/unix/native/libawt_xawt/awt/fp_pipewire.h

@@ -58,7 +58,7 @@ void (*fp_pw_stream_destroy)(struct pw_stream *stream);
 
 
 void (*fp_pw_init)(int *argc, char **argv[]);
-
+void (*fp_pw_deinit)(void);
 
 struct pw_core *
 (*fp_pw_context_connect_fd)(struct pw_context *context,

src/java.desktop/unix/native/libawt_xawt/awt/screencast_pipewire.c

@@ -89,12 +89,17 @@ static gboolean initScreenSpace() {
 }
 
 static void doCleanup() {
+    if (pw.loop) {
+        DEBUG_SCREENCAST("STOPPING loop\n", NULL);
+        fp_pw_thread_loop_stop(pw.loop);
+    }
+
     for (int i = 0; i < screenSpace.screenCount; ++i) {
         struct ScreenProps *screenProps = &screenSpace.screens[i];
         if (screenProps->data) {
             if (screenProps->data->stream) {
-                fp_pw_stream_disconnect(screenProps->data->stream);
                 fp_pw_thread_loop_lock(pw.loop);
+                fp_pw_stream_disconnect(screenProps->data->stream);
                 fp_pw_stream_destroy(screenProps->data->stream);
                 fp_pw_thread_loop_unlock(pw.loop);
                 screenProps->data->stream = NULL;
@@ -116,10 +121,7 @@ static void doCleanup() {
         pw.core = NULL;
     }
 
-    DEBUG_SCREENCAST("STOPPING loop\n", NULL)
-
     if (pw.loop) {
-        fp_pw_thread_loop_stop(pw.loop);
         fp_pw_thread_loop_destroy(pw.loop);
         pw.loop = NULL;
     }
@@ -130,6 +132,10 @@ static void doCleanup() {
         screenSpace.screenCount = 0;
     }
 
+    if (!sessionClosed) {
+        fp_pw_deinit();
+    }
+
     gtk->g_string_set_size(activeSessionToken, 0);
     sessionClosed = TRUE;
 }
@@ -571,6 +577,7 @@ static const struct pw_core_events coreEvents = {
  * @return TRUE on success
  */
 static gboolean doLoop(GdkRectangle requestedArea) {
+    gboolean isLoopLockTaken = FALSE;
     if (!pw.loop && !sessionClosed) {
         pw.loop = fp_pw_thread_loop_new("AWT Pipewire Thread", NULL);
 
@@ -599,6 +606,7 @@ static gboolean doLoop(GdkRectangle requestedArea) {
         }
 
         fp_pw_thread_loop_lock(pw.loop);
+        isLoopLockTaken = TRUE;
 
         pw.core = fp_pw_context_connect_fd(
                 pw.context,
@@ -639,12 +647,16 @@ static gboolean doLoop(GdkRectangle requestedArea) {
         DEBUG_SCREEN_PREFIX(screen, "@@@ screen processed %i\n", i);
     }
 
-    fp_pw_thread_loop_unlock(pw.loop);
+    if (isLoopLockTaken) {
+        fp_pw_thread_loop_unlock(pw.loop);
+    }
 
     return TRUE;
 
     fail:
-        fp_pw_thread_loop_unlock(pw.loop);
+        if (isLoopLockTaken) {
+            fp_pw_thread_loop_unlock(pw.loop);
+        }
         doCleanup();
         return FALSE;
 }
@@ -700,6 +712,7 @@ static gboolean loadSymbols() {
     LOAD_SYMBOL(fp_pw_stream_disconnect, "pw_stream_disconnect");
     LOAD_SYMBOL(fp_pw_stream_destroy, "pw_stream_destroy");
     LOAD_SYMBOL(fp_pw_init, "pw_init");
+    LOAD_SYMBOL(fp_pw_deinit, "pw_deinit");
     LOAD_SYMBOL(fp_pw_context_connect_fd, "pw_context_connect_fd");
     LOAD_SYMBOL(fp_pw_core_disconnect, "pw_core_disconnect");
     LOAD_SYMBOL(fp_pw_context_new, "pw_context_new");
@@ -855,6 +868,33 @@ static void arrayToRectangles(JNIEnv *env,
     (*env)->ReleaseIntArrayElements(env, boundsArray, body, 0);
 }
 
+static int makeScreencast(
+        const gchar *token,
+        GdkRectangle *requestedArea,
+        GdkRectangle *affectedScreenBounds,
+        gint affectedBoundsLength
+) {
+    if (!initScreencast(token, affectedScreenBounds, affectedBoundsLength)) {
+        return pw.pwFd;
+    }
+
+    if (!doLoop(*requestedArea)) {
+        return RESULT_ERROR;
+    }
+
+    while (!isAllDataReady()) {
+        fp_pw_thread_loop_lock(pw.loop);
+        fp_pw_thread_loop_wait(pw.loop);
+        fp_pw_thread_loop_unlock(pw.loop);
+        if (hasPipewireFailed) {
+            doCleanup();
+            return RESULT_ERROR;
+        }
+    }
+
+    return RESULT_OK;
+}
+
 /*
  * Class:     sun_awt_screencast_ScreencastHelper
  * Method:    closeSession
@@ -911,26 +951,22 @@ JNIEXPORT jint JNICALL Java_sun_awt_screencast_ScreencastHelper_getRGBPixelsImpl
             jx, jy, jwidth, jheight, token
     );
 
-    if (!initScreencast(token, affectedScreenBounds, affectedBoundsLength)) {
+    int attemptResult = makeScreencast(
-        releaseToken(env, jtoken, token);
+        token, &requestedArea, affectedScreenBounds, affectedBoundsLength);
-        return pw.pwFd;
-    }
 
-    if (!doLoop(requestedArea)) {
+    if (attemptResult) {
-        releaseToken(env, jtoken, token);
+        if (attemptResult == RESULT_DENIED) {
-        return RESULT_ERROR;
-    }
-
-    while (!isAllDataReady()) {
-        fp_pw_thread_loop_lock(pw.loop);
-        fp_pw_thread_loop_wait(pw.loop);
-        if (hasPipewireFailed) {
-            fp_pw_thread_loop_unlock(pw.loop);
-            doCleanup();
             releaseToken(env, jtoken, token);
-            return RESULT_ERROR;
+            return attemptResult;
+        }
+        DEBUG_SCREENCAST("Screencast attempt failed with %i, re-trying...\n",
+                         attemptResult);
+        attemptResult = makeScreencast(
+            token, &requestedArea, affectedScreenBounds, affectedBoundsLength);
+        if (attemptResult) {
+            releaseToken(env, jtoken, token);
+            return attemptResult;
         }
-        fp_pw_thread_loop_unlock(pw.loop);
     }
 
     DEBUG_SCREENCAST("\nall data ready\n", NULL);