obs-ffmpeg: Release encode texture early

During high graphics thread pressure it can take a significant time to
acquire the graphics lock. This change releases the OpenGL texture after
rendering to avoid the 2nd lock after sending the frame to FFmpeg. This
improves 99%-tile/100%-tile and median encode in a near encoder overload
scenario, and modestly raises the ceiling before encoder overload in my
test scene.

Master:
min=0 ms, median=4.29 ms, max=33.072 ms, 99th percentile=8.877 ms
min=0 ms, median=4.438 ms, max=77.157 ms, 99th percentile=9.853 ms
min=0 ms, median=4.527 ms, max=57.292 ms, 99th percentile=9.282 ms

This commit:
min=0.97 ms, median=3.009 ms, max=13.215 ms, 99th percentile=5.899 ms
min=1.181 ms, median=2.91 ms, max=9.854 ms, 99th percentile=5.56 ms
min=0.461 ms, median=3.013 ms, max=10.693 ms, 99th percentile=5.871 ms

plugins/obs-ffmpeg/obs-ffmpeg-vaapi.c

@@ -410,17 +410,13 @@ static inline enum gs_color_format drm_to_gs_color_format(int format)
 
 static void vaapi_destroy_surface(struct vaapi_surface *out)
 {
-	obs_enter_graphics();
-
 	for (uint32_t i = 0; i < out->num_textures; ++i) {
 		if (out->textures[i]) {
 			gs_texture_destroy(out->textures[i]);
 			out->textures[i] = NULL;
 		}
 	}
 
-	obs_leave_graphics();
-
 	av_frame_free(&out->frame);
 }
 
@@ -463,8 +459,6 @@ static bool vaapi_create_surface(struct vaapi_encoder *enc,
 		goto fail;
 	}
 
-	obs_enter_graphics();
-
 	for (uint32_t i = 0; i < desc.num_layers; ++i) {
 		unsigned int width = desc.width;
 		unsigned int height = desc.height;
@@ -489,8 +483,6 @@ static bool vaapi_create_surface(struct vaapi_encoder *enc,
 		out->num_textures++;
 	}
 
-	obs_leave_graphics();
-
 	for (uint32_t i = 0; i < desc.num_objects; ++i)
 		close(desc.objects[i].fd);
 
@@ -591,11 +583,11 @@ static inline bool vaapi_test_texencode(struct vaapi_encoder *enc)
 	    !obs_encoder_gpu_scaling_enabled(enc->encoder))
 		return false;
 
-	if (!vaapi_create_surface(enc, &surface))
-		return false;
-
+	obs_enter_graphics();
+	bool success = vaapi_create_surface(enc, &surface);
 	vaapi_destroy_surface(&surface);
-	return true;
+	obs_leave_graphics();
+	return success;
 }
 
 static void *vaapi_create_tex_internal(obs_data_t *settings,
@@ -840,19 +832,23 @@ static bool vaapi_encode_tex(void *data, struct encoder_texture *texture,
 
 	*received_packet = false;
 
+	obs_enter_graphics();
+
+	// Destroyed piecemeal to avoid taking the graphics lock again.
 	if (!vaapi_create_surface(enc, &surface)) {
 		warn("vaapi_encode_tex: failed to create texture hw frame");
+		obs_leave_graphics();
 		return false;
 	}
 
-	obs_enter_graphics();
-
 	for (uint32_t i = 0; i < surface.num_textures; ++i) {
 		if (!texture->tex[i]) {
 			warn("vaapi_encode_tex: unexpected number of textures");
+			obs_leave_graphics();
 			goto fail;
 		}
 		gs_copy_texture(surface.textures[i], texture->tex[i]);
+		gs_texture_destroy(surface.textures[i]);
 	}
 
 	gs_flush();
@@ -871,11 +867,11 @@ static bool vaapi_encode_tex(void *data, struct encoder_texture *texture,
 	if (!vaapi_encode_internal(enc, surface.frame, packet, received_packet))
 		goto fail;
 
-	vaapi_destroy_surface(&surface);
+	av_frame_free(&surface.frame);
 	return true;
 
 fail:
-	vaapi_destroy_surface(&surface);
+	av_frame_free(&surface.frame);
 	return false;
 }