From 0124ebe43bee0dec438da2deb11ea39d9ade3bbe Mon Sep 17 00:00:00 2001 From: Madiator2011 Date: Tue, 12 Nov 2024 19:04:37 +0100 Subject: [PATCH] Fixes for crash during install process, reduced logs spam for Fileapp check --- .../better-ai-launcher/app/app.py | 81 ++++-- .../better-ai-launcher/app/gunicorn.conf.py | 2 +- .../app/templates/index.html | 233 ++++++++++-------- .../better-ai-launcher/app/utils/app_utils.py | 52 +--- .../app/utils/filebrowser_utils.py | 10 +- 5 files changed, 205 insertions(+), 173 deletions(-) diff --git a/official-templates/better-ai-launcher/app/app.py b/official-templates/better-ai-launcher/app/app.py index 99975d6..82488d1 100644 --- a/official-templates/better-ai-launcher/app/app.py +++ b/official-templates/better-ai-launcher/app/app.py @@ -223,40 +223,54 @@ def force_kill_app(app_name): except Exception as e: return jsonify({'status': 'error', 'message': str(e)}) +from gevent.lock import RLock +websocket_lock = RLock() + @sock.route('/ws') def websocket(ws): - active_websockets.add(ws) + with websocket_lock: + active_websockets.add(ws) try: - while True: - message = ws.receive() - data = json.loads(message) - - if data['type'] == 'heartbeat': - ws.send(json.dumps({'type': 'heartbeat'})) - else: - # Handle other message types - pass + while ws.connected: # Check connection status + try: + message = ws.receive(timeout=70) # Add timeout slightly higher than heartbeat + if message: + data = json.loads(message) + if data['type'] == 'heartbeat': + ws.send(json.dumps({'type': 'heartbeat'})) + else: + # Handle other message types + pass + except Exception as e: + if "timed out" in str(e).lower(): + # Handle timeout gracefully + continue + print(f"Error handling websocket message: {str(e)}") + if not ws.connected: + break + continue except Exception as e: print(f"WebSocket error: {str(e)}") finally: - active_websockets.remove(ws) + with websocket_lock: + try: + active_websockets.remove(ws) + except KeyError: + pass def send_heartbeat(): - initial_interval = 5 # 5 seconds - max_interval = 60 # 60 seconds - current_interval = initial_interval - start_time = time.time() - while True: - time.sleep(current_interval) - send_websocket_message('heartbeat', {}) - - # Gradually increase the interval - elapsed_time = time.time() - start_time - if elapsed_time < 60: # First minute - current_interval = min(current_interval * 1.5, max_interval) - else: - current_interval = max_interval + try: + time.sleep(60) # Fixed 60 second interval + with websocket_lock: + for ws in list(active_websockets): # Create a copy of the set + try: + if ws.connected: + ws.send(json.dumps({'type': 'heartbeat', 'data': {}})) + except Exception as e: + print(f"Error sending heartbeat: {str(e)}") + except Exception as e: + print(f"Error in heartbeat thread: {str(e)}") # Start heartbeat thread threading.Thread(target=send_heartbeat, daemon=True).start() @@ -264,7 +278,15 @@ threading.Thread(target=send_heartbeat, daemon=True).start() @app.route('/install/', methods=['POST']) def install_app_route(app_name): try: - success, message = install_app(app_name, app_configs, send_websocket_message) + def progress_callback(message_type, message_data): + try: + send_websocket_message(message_type, message_data) + except Exception as e: + print(f"Error sending progress update: {str(e)}") + # Continue even if websocket fails + pass + + success, message = install_app(app_name, app_configs, progress_callback) if success: return jsonify({'status': 'success', 'message': message}) else: @@ -335,7 +357,12 @@ def stop_filebrowser_route(): @app.route('/filebrowser_status') def filebrowser_status_route(): - return jsonify({'status': get_filebrowser_status()}) + try: + status = get_filebrowser_status() + return jsonify({'status': status if status else 'unknown'}) + except Exception as e: + app.logger.error(f"Error getting filebrowser status: {str(e)}") + return jsonify({'status': 'error', 'message': str(e)}), 500 @app.route('/add_app_config', methods=['POST']) def add_new_app_config(): diff --git a/official-templates/better-ai-launcher/app/gunicorn.conf.py b/official-templates/better-ai-launcher/app/gunicorn.conf.py index e6f47fe..902b82c 100644 --- a/official-templates/better-ai-launcher/app/gunicorn.conf.py +++ b/official-templates/better-ai-launcher/app/gunicorn.conf.py @@ -5,7 +5,7 @@ import multiprocessing workers = multiprocessing.cpu_count() * 2 + 1 worker_class = 'geventwebsocket.gunicorn.workers.GeventWebSocketWorker' worker_connections = 1000 -timeout = 300 +timeout = 0 # Disable timeout completely # Server socket bind = '0.0.0.0:7222' diff --git a/official-templates/better-ai-launcher/app/templates/index.html b/official-templates/better-ai-launcher/app/templates/index.html index b0f08db..167d9eb 100644 --- a/official-templates/better-ai-launcher/app/templates/index.html +++ b/official-templates/better-ai-launcher/app/templates/index.html @@ -2640,6 +2640,8 @@ document.getElementById('loadingOverlay').style.display = 'none'; reconnectAttempts = 0; initializeUI(); + // Start sending heartbeats immediately after connection + startHeartbeat(); }; socket.onmessage = function(event) { @@ -2648,7 +2650,8 @@ console.log(`Data received from server:`, data); if (data.type === 'heartbeat') { - sendWebSocketMessage('heartbeat', {}); + // Reset heartbeat timeout on receiving heartbeat response + resetHeartbeatTimeout(); } else if (data.type === 'model_download_progress') { updateModelDownloadProgress(data.data); } else if (data.type === 'status_update') { @@ -2675,14 +2678,70 @@ }; } + let heartbeatInterval; + let heartbeatTimeout; + + function startHeartbeat() { + // Clear any existing intervals + if (heartbeatInterval) { + clearInterval(heartbeatInterval); + } + + // Send heartbeat every 60 seconds + heartbeatInterval = setInterval(() => { + if (socket.readyState === WebSocket.OPEN) { + socket.send(JSON.stringify({ type: 'heartbeat' })); + // Set timeout to reconnect if no response within 10 seconds + setHeartbeatTimeout(); + } + }, 60000); + } + + function setHeartbeatTimeout() { + if (heartbeatTimeout) { + clearTimeout(heartbeatTimeout); + } + heartbeatTimeout = setTimeout(() => { + console.log("Heartbeat timeout - attempting reconnect..."); + if (socket) { + socket.close(); + } + handleReconnect(); + }, 65000); // Increased timeout to 65 seconds + } + + function resetHeartbeatTimeout() { + if (heartbeatTimeout) { + clearTimeout(heartbeatTimeout); + } + } + + // Clean up on page unload + window.addEventListener('beforeunload', () => { + if (heartbeatInterval) { + clearInterval(heartbeatInterval); + } + if (heartbeatTimeout) { + clearTimeout(heartbeatTimeout); + } + if (socket) { + socket.close(); + } + }); + function handleReconnect() { if (reconnectAttempts < maxReconnectAttempts) { reconnectAttempts++; document.getElementById('loadingOverlay').style.display = 'flex'; - document.getElementById('loadingMessage').textContent = `WebSocket disconnected. Attempting to reconnect (${reconnectAttempts}/${maxReconnectAttempts})...`; - setTimeout(connectWebSocket, reconnectInterval); + document.getElementById('loadingMessage').textContent = + `WebSocket disconnected. Attempting to reconnect (${reconnectAttempts}/${maxReconnectAttempts})...`; + + // Add exponential backoff + const backoffTime = Math.min(1000 * Math.pow(2, reconnectAttempts), 30000); + setTimeout(connectWebSocket, backoffTime); } else { - document.getElementById('loadingMessage').textContent = 'Failed to connect to WebSocket. Please refresh the page or contact support.'; + document.getElementById('loadingMessage').textContent = + 'Failed to connect to WebSocket. Installation continues in background. Please refresh page to see status.'; } } @@ -2873,9 +2932,14 @@ setInterval(updateStatus, 5000); setInterval(updateLogs, 1000); + // Initialize model types + refreshModelTypes(); + } + + async function refreshModelTypes() { var data = {}; data.cmd = 'refreshModelTypes'; - extendUIHelper(data); // lutzapps - initialize the available SHARED_MODEL_FOLDERS for the "Model Downloader" modelType select list + await extendUIHelper(data); // lutzapps - initialize the available SHARED_MODEL_FOLDERS for the "Model Downloader" modelType select list } // lutzapps - populate modeltype select options from shared_models @@ -2933,53 +2997,49 @@ case "refreshModelTypes": // refresh and optionally select the 'modelType' list for "Model Downloader" var modelTypeSelect = document.getElementById('modelType'); - - // get the data from the Server - response = await fetch('/get_model_types'); - result = await response.json(); - - //alert(JSON.stringify(result)); // show the JSON-String - var model_types = result; // get the JSON-Object - var count = Object.keys(model_types).length; // count=18, when using the default SHARED_MODEL_FOLDERS dict - - // the "/get_model_types" app.get_model_types_route() function checks - // if the SHARED_MODELS_DIR shared files already exists at the "/workspace" location. - // that only happens AFTER the the user clicked the "Create Shared Folders" button - // on the "Settings" Tab of the app's WebUI. - // it will return an empty model_types_dict, so the "Download Manager" does NOT get - // the already in-memory SHARED_MODEL_FOLDERS code-generated default dict - // BEFORE the workspace folders in SHARED_MODELS_DIR exists! - // - // when SHARED_MODELS_DIR exists (or updates), this function will be called via a Socket Message - // to "refresh" its content automatically - - var modelTypeSelected = modelTypeSelect.value; // remember the current selected modelType.option value - modelTypeSelect.options.length = 0; // clear all current modelTypeSelect options - - for (i = 0 ; i < count; i += 1) { - modelTypeOption = document.createElement('option'); - - modelType = model_types[String(i)]['modelfolder']; - modelTypeOption.setAttribute('value', modelType); - modelTypeOption.appendChild(document.createTextNode(model_types[String(i)]['desc'])); - //if (modelFolder === modelTypeSelected) { - // modelTypeOption.selected = true; // reselect it - //} - - modelTypeSelect.appendChild(modelTypeOption); + if (!modelTypeSelect) { + console.error('Model type select element not found'); + return; } - //modelTypeSelect.selectedIndex = modelfolder_index; // set the selected index - //modelTypeSelect.options[mmodelfolder_index].selected = true; // and mark it as "selected" option - if (modelTypeSelected === "") { // initial refresh, called by initializeUI() function - modelTypeSelect.selectedIndex = 0; // use the first modelType option, usually "ckpt" - MODELTYPE_SELECTED = modelTypeSelect.options[0].value; // NOT handled by the onchange() event handler - } - else { - modelTypeSelect.value = modelTypeSelected; // (re-)apply the selected modelType option - MODELTYPE_SELECTED = modelTypeSelected; // NOT handled by the onchange() event handler - } + try { + // get the data from the Server + response = await fetch('/get_model_types'); + result = await response.json(); + var model_types = result; // get the JSON-Object + var count = Object.keys(model_types).length; + + var modelTypeSelected = modelTypeSelect.value; // remember the current selected modelType.option value + modelTypeSelect.options.length = 0; // clear all current modelTypeSelect options + + if (count === 0) { + // Add a default option if no model types are available + const defaultOption = document.createElement('option'); + defaultOption.text = 'Please create shared folders first'; + defaultOption.value = ''; + modelTypeSelect.add(defaultOption); + return; + } + + for (i = 0; i < count; i++) { + modelTypeOption = document.createElement('option'); + modelType = model_types[String(i)]['modelfolder']; + modelTypeOption.setAttribute('value', modelType); + modelTypeOption.appendChild(document.createTextNode(model_types[String(i)]['desc'])); + modelTypeSelect.appendChild(modelTypeOption); + } + + if (modelTypeSelected === "") { + modelTypeSelect.selectedIndex = 0; + MODELTYPE_SELECTED = modelTypeSelect.options[0].value; + } else { + modelTypeSelect.value = modelTypeSelected; + MODELTYPE_SELECTED = modelTypeSelected; + } + } catch (error) { + console.error('Error refreshing model types:', error); + } break; case "selectModelType": @@ -3074,7 +3134,7 @@ progressBar.style.width = '100%'; progressBar.textContent = '100%'; await loadModelFolders(); - showRecreateSymlinksButton(); + // Remove showRecreateSymlinksButton call since it's automatic now } else if (result.status === 'choice_required') { if (result.data.type === 'file') { showFileChoiceDialog(result.data, url, modelName, modelType, civitaiToken, hfToken); @@ -3175,31 +3235,6 @@ } } - function showRecreateSymlinksButton() { - const buttonContainer = document.getElementById('recreate-symlinks-container'); - buttonContainer.innerHTML = ` - - - `; - buttonContainer.style.display = 'block'; - } - - async function recreateSymlinks() { - const statusElement = document.getElementById('symlink-status'); - statusElement.textContent = 'Recreating symlinks...'; - try { - const response = await fetch('/recreate_symlinks', { method: 'POST' }); - const data = await response.json(); - if (data.status === 'success') { - statusElement.textContent = data.message; - } else { - statusElement.textContent = 'Error: ' + data.message; - } - } catch (error) { - statusElement.textContent = 'Error: ' + error.message; - } - } - async function saveCivitaiToken() { const token = document.getElementById('civitaiTokenSave').value; try { @@ -3315,12 +3350,10 @@ // Call this function when the Models tab is opened document.querySelector('.navbar-tabs a[onclick="openTab(event, \'models-tab\')"]').addEventListener('click', function() { - //alert("querySelector"); - loadModelFolders(); // lutzapps - this ModelFolders is NOT for the 'modelType' "select dropdown" model list - extendUIHelper(); // lutzapps - select the last know MODELTYPE_SELECTED in the WebUI Dom Id 'modelType' "select dropdown" model list + loadModelFolders(); + refreshModelTypes(); // Refresh model types when switching to Models tab loadCivitaiToken(); - loadHFToken(); // lutzapps - added HF_TOKEN ENV var Support - + loadHFToken(); updateExampleUrls(); }); @@ -3467,27 +3500,26 @@ } } - async function updateFileBrowserStatus() { - try { - const response = await fetch('/filebrowser_status'); - const result = await response.json(); - const statusElement = document.getElementById('filebrowser-status'); - if (statusElement) { - statusElement.textContent = result.status; - } - const startButton = document.getElementById('start-filebrowser'); - const stopButton = document.getElementById('stop-filebrowser'); - if (startButton && stopButton) { - startButton.disabled = (result.status === 'running'); - stopButton.disabled = (result.status === 'stopped'); - } - } catch (error) { - console.error('Error updating File Browser status:', error); - } + function updateFileBrowserStatus() { + fetch('/filebrowser_status') + .then(response => response.json()) + .then(data => { + const statusElement = document.getElementById('filebrowser-status'); + if (statusElement) { + statusElement.textContent = data.status; + } + const startButton = document.getElementById('start-filebrowser'); + const stopButton = document.getElementById('stop-filebrowser'); + if (startButton && stopButton) { + startButton.disabled = (data.status === 'running'); + stopButton.disabled = (data.status === 'stopped'); + } + }) + .catch(error => console.error('Error updating File Browser status:', error)); } - // Call this function periodically to update the status - setInterval(updateFileBrowserStatus, 5000); + // Reduce the frequency of status updates + setInterval(updateFileBrowserStatus, 30000); // Check every 30 seconds instead of 5 seconds // Update the DOMContentLoaded event listener document.addEventListener('DOMContentLoaded', function() { @@ -3542,7 +3574,8 @@ console.log("WebSocket message received:", data); if (data.type === 'heartbeat') { - sendWebSocketMessage('heartbeat', {}); + // Reset heartbeat timeout on receiving heartbeat response + resetHeartbeatTimeout(); } else if (data.type === 'install_progress') { updateInstallProgress(data.data); } else if (data.type === 'install_log') { diff --git a/official-templates/better-ai-launcher/app/utils/app_utils.py b/official-templates/better-ai-launcher/app/utils/app_utils.py index c846a56..abc519c 100644 --- a/official-templates/better-ai-launcher/app/utils/app_utils.py +++ b/official-templates/better-ai-launcher/app/utils/app_utils.py @@ -699,39 +699,12 @@ def download_and_unpack_venv_v2(app_name:str, app_configs:dict, send_websocket_m return False, error_message send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 100, 'stage': 'Unpacking Complete'}) + send_websocket_message('install_log', {'app_name': app_name, 'log': 'Unpacking complete. Proceeding to clone repository...'}) - ### installing the App from GITHUB - # Clone the repository if it doesn't exist - success, message = clone_application(app_name) - - print(f"'DEBUG_SETTINGS' after this run:\n{pretty_dict(DEBUG_SETTINGS)}") - - ### original "v1" code (very slow code because of STATISTICS glory - - # unpack_command = f"tar -xzvf {downloaded_file} -C {venv_path}" - # process = subprocess.Popen(unpack_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) - - # total_files = sum(1 for _ in subprocess.Popen(f"tar -tvf {downloaded_file}", shell=True, stdout=subprocess.PIPE).stdout) - # files_processed = 0 - - # for line in process.stdout: - # files_processed += 1 - # percentage = min(int((files_processed / total_files) * 100), 100) - # send_websocket_message('install_progress', { - # 'app_name': app_name, - # 'percentage': percentage, - # 'stage': 'Unpacking', - # 'processed': files_processed, - # 'total': total_files - # }) - # send_websocket_message('install_log', {'app_name': app_name, 'log': f"Unpacking: {line.strip()}"}) - - # process.wait() - # rc = process.returncode - - ### installing the App from GITHUB - # Clone the repository if it doesn't exist - success, error_message = clone_application(app_name, send_websocket_message) + # Clone the repository + success, message = clone_application(app_config, send_websocket_message) + if not success: + return False, message # Clean up the downloaded file send_websocket_message('install_log', {'app_name': app_name, 'log': 'Cleaning up...'}) @@ -742,19 +715,10 @@ def download_and_unpack_venv_v2(app_name:str, app_configs:dict, send_websocket_m os.remove(downloaded_file) send_websocket_message('install_log', {'app_name': app_name, 'log': 'Installation complete. Refresh page to start app'}) + save_install_status(app_name, 'completed', 100, 'Completed') + send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."}) + return True, "Virtual environment installed successfully." - if success: - save_install_status(app_name, 'completed', 100, 'Completed') - send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."}) - return True, "Virtual environment installed successfully." - else: - return False, error_message - - except requests.RequestException as e: - error_message = f"Download/Decompression failed: {str(e)}" - send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message}) - save_install_status(app_name, 'failed', 0, 'Failed') - return False, error_message except Exception as e: error_message = f"Installation failed: {str(e)}\n{traceback.format_exc()}" save_install_status(app_name, 'failed', 0, 'Failed') diff --git a/official-templates/better-ai-launcher/app/utils/filebrowser_utils.py b/official-templates/better-ai-launcher/app/utils/filebrowser_utils.py index f5a94e4..d83e644 100644 --- a/official-templates/better-ai-launcher/app/utils/filebrowser_utils.py +++ b/official-templates/better-ai-launcher/app/utils/filebrowser_utils.py @@ -1,5 +1,7 @@ import subprocess import time +import requests +from requests.exceptions import Timeout FILEBROWSER_PORT = 8181 filebrowser_process = None @@ -35,4 +37,10 @@ def stop_filebrowser(): return False def get_filebrowser_status(): - return 'running' if filebrowser_process and filebrowser_process.poll() is None else 'stopped' + try: + response = requests.get('http://localhost:7222/fileapp/', timeout=5) + return 'running' if response.status_code == 200 else 'stopped' + except Timeout: + return 'timeout' + except Exception: + return 'unknown'