feat: Implement lazy loading for core modules and enhance process handling in Python API

These changes from this commit and the last implement huge performance improvements. Will allow for multiple processes to happen at once as well.

Author: SirStig

EncodeForge/src/main/java/com/encodeforge/MainApp.java

@@ -4,6 +4,7 @@
 import com.encodeforge.controller.MainController;
 import com.encodeforge.service.DependencyManager;
 import com.encodeforge.service.PythonBridge;
+import com.encodeforge.service.PythonProcessPool;
 import com.encodeforge.util.PathManager;
 import com.encodeforge.util.ResourceExtractor;
 import javafx.application.Application;
@@ -31,9 +32,11 @@ public class MainApp extends Application {
     private static final String VERSION = "0.3.1";
 
     private DependencyManager dependencyManager;
-    private PythonBridge pythonBridge;
+    private PythonBridge pythonBridge; // Legacy - will be replaced by processPool
+    private PythonProcessPool processPool;
     private MainController controller;
     private boolean pendingDependencyInstall = false;
+    private boolean useProcessPool = true; // Feature flag for gradual rollout
 
     @Override
     public void init() throws Exception {
@@ -103,14 +106,31 @@ public void init() throws Exception {
     }
 
     /**
-     * Initialize runtime components (Python bridge, etc.)
+     * Initialize runtime components (Python bridge or process pool)
      */
     private void initializeRuntime() throws IOException {
         try {
-            // Initialize Python bridge (will use bundled or system Python)
-            pythonBridge = new PythonBridge(dependencyManager);
-            pythonBridge.start();
-            logger.info("Python bridge initialized");
+            if (useProcessPool) {
+                // NEW: Multi-process pool for concurrent operations
+                logger.info("Initializing Python process pool (multi-worker mode)");
+                processPool = new PythonProcessPool(dependencyManager, 4); // 4 workers for good balance
+                
+                // Start asynchronously - UI won't be blocked
+                processPool.startAsync()
+                    .thenRun(() -> logger.info("Python process pool started successfully"))
+                    .exceptionally(error -> {
+                        logger.error("Failed to start process pool", error);
+                        return null;
+                    });
+                
+                logger.info("Python process pool initialization started (async)");
+            } else {
+                // LEGACY: Single Python bridge (backwards compatibility)
+                logger.info("Initializing Python bridge (legacy single-worker mode)");
+                pythonBridge = new PythonBridge(dependencyManager);
+                pythonBridge.start();
+                logger.info("Python bridge initialized");
+            }
         } catch (IOException e) {
             // Check if this is a Python not found error
             if (e.getMessage() != null && e.getMessage().contains("Python not found")) {
@@ -178,7 +198,13 @@ public void start(Stage primaryStage) {
             FXMLLoader loader = new FXMLLoader(getClass().getResource("/fxml/MainView.fxml"));
 
             // Create controller and inject dependencies
-            controller = new MainController(pythonBridge);
+            if (useProcessPool) {
+                // NEW: Pass process pool to controller
+                controller = new MainController(processPool);
+            } else {
+                // LEGACY: Pass Python bridge to controller
+                controller = new MainController(pythonBridge);
+            }
             loader.setController(controller);
 
             // Load scene
@@ -267,7 +293,11 @@ public void stop() throws Exception {
             controller.shutdown();
         }
 
-        if (pythonBridge != null) {
+        if (useProcessPool && processPool != null) {
+            logger.info("Shutting down Python process pool");
+            processPool.shutdown();
+        } else if (pythonBridge != null) {
+            logger.info("Shutting down Python bridge");
             pythonBridge.shutdown();
         }
 

EncodeForge/src/main/java/com/encodeforge/controller/MainController.java

@@ -4,6 +4,7 @@
 import com.encodeforge.model.ConversionSettings;
 import com.encodeforge.service.DependencyManager;
 import com.encodeforge.service.PythonBridge;
+import com.encodeforge.service.PythonProcessPool;
 import com.encodeforge.util.HardwareDetector;
 import com.encodeforge.util.PathManager;
 import com.encodeforge.util.StatusManager;
@@ -49,14 +50,16 @@
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Main window controller for the modernized UI
  */
 public class MainController {
     private static final Logger logger = LoggerFactory.getLogger(MainController.class);
 
-    private final PythonBridge pythonBridge;
+    private final PythonBridge pythonBridge; // Legacy mode
+    private final PythonProcessPool processPool; // Multi-worker mode
     private final DependencyManager dependencyManager;
     private final StatusManager statusManager;
     // Separate queues for different states
@@ -299,8 +302,37 @@ private enum ResizeDirection {
     @FXML private Tab fileInfoTab;
     @FXML private Tab logsTab;
 
+    /**
+     * Constructor for legacy single-worker mode
+     */
     public MainController(PythonBridge pythonBridge) {
         this.pythonBridge = pythonBridge;
+        this.processPool = null; // Legacy mode
+        
+        // Get dependency manager from MainApp (will be passed in future refactor)
+        // For now, create a new one
+        try {
+            this.dependencyManager = new DependencyManager();
+        } catch (IOException e) {
+            logger.error("Failed to initialize DependencyManager", e);
+            throw new RuntimeException("Could not initialize dependency manager", e);
+        }
+        
+        this.statusManager = StatusManager.getInstance();
+        
+        // Load settings from disk
+        this.settings = ConversionSettings.load();
+        logger.info("Settings loaded from: {}", ConversionSettings.getSettingsFilePath());
+        
+        logger.info("MainController initialized with PythonBridge (legacy mode)");
+    }
+    
+    /**
+     * Constructor for new multi-worker mode
+     */
+    public MainController(PythonProcessPool processPool) {
+        this.pythonBridge = null; // Multi-worker mode
+        this.processPool = processPool;
 
         // Get dependency manager from MainApp (will be passed in future refactor)
         // For now, create a new one
@@ -317,7 +349,7 @@ public MainController(PythonBridge pythonBridge) {
         this.settings = ConversionSettings.load();
         logger.info("Settings loaded from: {}", ConversionSettings.getSettingsFilePath());
 
-        // FFmpeg is now managed by DependencyManager, no need for embedded setup here
+        logger.info("MainController initialized with PythonProcessPool (multi-worker mode)");
     }
 
     @FXML
@@ -2532,8 +2564,137 @@ private void resetProcessingState() {
      * FFmpeg detection now uses Java-side DependencyManager instead of Python.
      */
     private void updateAllStatus() {
+        // Use parallel execution if process pool is available
+        if (processPool != null) {
+            updateAllStatusParallel();
+        } else {
+            updateAllStatusSequential();
+        }
+    }
+    
+    /**
+     * Parallel version using PythonProcessPool for fast concurrent checks
+     */
+    private void updateAllStatusParallel() {
+        try {
+            logger.info("Starting parallel status check (multi-process mode)");
+            long startTime = System.currentTimeMillis();
+            
+            // Sync settings with Python backend (quick operation)
+            try {
+                JsonObject settingsParams = new JsonObject();
+                settingsParams.add("settings", settings.toJson());
+                processPool.submitQuickTask("update_settings", settingsParams).get(2, TimeUnit.SECONDS);
+                logger.info("Settings synchronized with Python backend");
+            } catch (Exception e) {
+                logger.error("Failed to sync settings with Python backend", e);
+            }
+            
+            // Check FFmpeg using Java DependencyManager (parallel with Python checks)
+            CompletableFuture<Boolean> ffmpegCheckFuture = CompletableFuture.supplyAsync(() -> {
+                try {
+                    boolean available = dependencyManager.checkFFmpeg().get();
+                    logger.info("FFmpeg check completed: {}", available);
+                    return available;
+                } catch (Exception e) {
+                    logger.error("Error checking FFmpeg via DependencyManager", e);
+                    return false;
+                }
+            });
+            
+            // Parallel Python backend checks
+            CompletableFuture<JsonObject> getAllStatusFuture = CompletableFuture.supplyAsync(() -> {
+                try {
+                    JsonObject result = processPool.submitQuickTask("get_all_status").get(5, TimeUnit.SECONDS);
+                    logger.info("get_all_status completed");
+                    return result;
+                } catch (Exception e) {
+                    logger.error("Error getting all status", e);
+                    JsonObject error = new JsonObject();
+                    error.addProperty("status", "error");
+                    error.addProperty("message", e.getMessage());
+                    return error;
+                }
+            });
+            
+            // Wait for all checks to complete
+            CompletableFuture<Void> allChecks = CompletableFuture.allOf(
+                ffmpegCheckFuture,
+                getAllStatusFuture
+            );
+            
+            allChecks.get(10, TimeUnit.SECONDS); // Wait for all checks
+            
+            // Get results
+            boolean ffmpegAvailable = ffmpegCheckFuture.get();
+            String ffmpegVersion = "Not Found";
+            
+            if (ffmpegAvailable) {
+                java.nio.file.Path ffmpegPath = dependencyManager.getInstalledFFmpegPath();
+                if (ffmpegPath != null) {
+                    ffmpegVersion = "Found";
+                    logger.info("FFmpeg detected via DependencyManager at: {}", ffmpegPath);
+                } else {
+                    ffmpegVersion = "System PATH";
+                    logger.info("FFmpeg detected in system PATH");
+                }
+            }
+            
+            JsonObject response = getAllStatusFuture.get();
+            
+            if (response.has("status") && !response.get("status").isJsonNull() 
+                && response.get("status").getAsString().equals("error")) {
+                String errorMsg = response.has("message") && !response.get("message").isJsonNull() 
+                    ? response.get("message").getAsString() 
+                    : "Unknown error";
+                logger.error("Error getting Python status: " + errorMsg);
+            }
+            
+            // Update FFmpeg info in response (override Python's check with Java's)
+            if (response.has("ffmpeg")) {
+                JsonObject ffmpegInfo = response.getAsJsonObject("ffmpeg");
+                ffmpegInfo.addProperty("available", ffmpegAvailable);
+                if (ffmpegAvailable && ffmpegInfo.has("version")) {
+                    ffmpegVersion = ffmpegInfo.get("version").getAsString();
+                }
+            }
+            
+            // Update StatusManager with the consolidated response
+            com.encodeforge.util.StatusManager.getInstance().updateFromResponse(response);
+            
+            // Update UI on JavaFX thread
+            final boolean finalFfmpegAvailable = ffmpegAvailable;
+            final String finalFfmpegVersion = ffmpegVersion;
+            
+            Platform.runLater(() -> {
+                updateFFmpegStatus(finalFfmpegAvailable, finalFfmpegVersion);
+                updateUIFromStatus();
+            });
+            
+            long elapsedMs = System.currentTimeMillis() - startTime;
+            logger.info("Parallel status check completed in {}ms", elapsedMs);
+            
+        } catch (TimeoutException e) {
+            logger.error("Status check timed out", e);
+            Platform.runLater(() -> {
+                log("ERROR: Status check timed out - " + e.getMessage());
+                updateFFmpegStatus(false, "Timeout");
+            });
+        } catch (Exception e) {
+            logger.error("Unexpected error in parallel status check", e);
+            Platform.runLater(() -> {
+                log("ERROR: " + e.getMessage());
+                updateFFmpegStatus(false, "Error");
+            });
+        }
+    }
+    
+    /**
+     * Sequential version using legacy PythonBridge (backward compatibility)
+     */
+    private void updateAllStatusSequential() {
         try {
-            logger.info("Starting consolidated status check");
+            logger.info("Starting consolidated status check (legacy mode)");
 
             // First, sync settings with Python backend
             try {
@@ -6394,7 +6555,13 @@ private void checkForOngoingConversions() {
         try {
             logger.info("Checking for ongoing conversions from previous session...");
 
-            JsonObject response = pythonBridge.checkOngoingConversion();
+            // Use appropriate bridge based on mode
+            JsonObject response;
+            if (processPool != null) {
+                response = processPool.submitQuickTask("check_ongoing_conversion").get(3, TimeUnit.SECONDS);
+            } else {
+                response = pythonBridge.checkOngoingConversion();
+            }
 
             if (response == null || !response.has("status")) {
                 logger.warn("Invalid response from check_ongoing_conversion");