mirror of
https://github.com/kodxana/madiator-docker-runpod.git
synced 2024-11-21 18:30:12 +01:00
Fixes for crash during install process, reduced logs spam for Fileapp check
This commit is contained in:
Madiator2011
GitHub
parent
0af6151735
commit
0124ebe43b
5 changed files with 205 additions and 173 deletions
|
|
@ -223,40 +223,54 @@ def force_kill_app(app_name):
|
|||
except Exception as e:
|
||||
return jsonify({'status': 'error', 'message': str(e)})
|
||||
|
||||
from gevent.lock import RLock
|
||||
websocket_lock = RLock()
|
||||
|
||||
@sock.route('/ws')
|
||||
def websocket(ws):
|
||||
active_websockets.add(ws)
|
||||
with websocket_lock:
|
||||
active_websockets.add(ws)
|
||||
try:
|
||||
while True:
|
||||
message = ws.receive()
|
||||
data = json.loads(message)
|
||||
|
||||
if data['type'] == 'heartbeat':
|
||||
ws.send(json.dumps({'type': 'heartbeat'}))
|
||||
else:
|
||||
# Handle other message types
|
||||
pass
|
||||
while ws.connected: # Check connection status
|
||||
try:
|
||||
message = ws.receive(timeout=70) # Add timeout slightly higher than heartbeat
|
||||
if message:
|
||||
data = json.loads(message)
|
||||
if data['type'] == 'heartbeat':
|
||||
ws.send(json.dumps({'type': 'heartbeat'}))
|
||||
else:
|
||||
# Handle other message types
|
||||
pass
|
||||
except Exception as e:
|
||||
if "timed out" in str(e).lower():
|
||||
# Handle timeout gracefully
|
||||
continue
|
||||
print(f"Error handling websocket message: {str(e)}")
|
||||
if not ws.connected:
|
||||
break
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f"WebSocket error: {str(e)}")
|
||||
finally:
|
||||
active_websockets.remove(ws)
|
||||
with websocket_lock:
|
||||
try:
|
||||
active_websockets.remove(ws)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def send_heartbeat():
|
||||
initial_interval = 5 # 5 seconds
|
||||
max_interval = 60 # 60 seconds
|
||||
current_interval = initial_interval
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
time.sleep(current_interval)
|
||||
send_websocket_message('heartbeat', {})
|
||||
|
||||
# Gradually increase the interval
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time < 60: # First minute
|
||||
current_interval = min(current_interval * 1.5, max_interval)
|
||||
else:
|
||||
current_interval = max_interval
|
||||
try:
|
||||
time.sleep(60) # Fixed 60 second interval
|
||||
with websocket_lock:
|
||||
for ws in list(active_websockets): # Create a copy of the set
|
||||
try:
|
||||
if ws.connected:
|
||||
ws.send(json.dumps({'type': 'heartbeat', 'data': {}}))
|
||||
except Exception as e:
|
||||
print(f"Error sending heartbeat: {str(e)}")
|
||||
except Exception as e:
|
||||
print(f"Error in heartbeat thread: {str(e)}")
|
||||
|
||||
# Start heartbeat thread
|
||||
threading.Thread(target=send_heartbeat, daemon=True).start()
|
||||
|
|
@ -264,7 +278,15 @@ threading.Thread(target=send_heartbeat, daemon=True).start()
|
|||
@app.route('/install/<app_name>', methods=['POST'])
|
||||
def install_app_route(app_name):
|
||||
try:
|
||||
success, message = install_app(app_name, app_configs, send_websocket_message)
|
||||
def progress_callback(message_type, message_data):
|
||||
try:
|
||||
send_websocket_message(message_type, message_data)
|
||||
except Exception as e:
|
||||
print(f"Error sending progress update: {str(e)}")
|
||||
# Continue even if websocket fails
|
||||
pass
|
||||
|
||||
success, message = install_app(app_name, app_configs, progress_callback)
|
||||
if success:
|
||||
return jsonify({'status': 'success', 'message': message})
|
||||
else:
|
||||
|
|
@ -335,7 +357,12 @@ def stop_filebrowser_route():
|
|||
|
||||
@app.route('/filebrowser_status')
|
||||
def filebrowser_status_route():
|
||||
return jsonify({'status': get_filebrowser_status()})
|
||||
try:
|
||||
status = get_filebrowser_status()
|
||||
return jsonify({'status': status if status else 'unknown'})
|
||||
except Exception as e:
|
||||
app.logger.error(f"Error getting filebrowser status: {str(e)}")
|
||||
return jsonify({'status': 'error', 'message': str(e)}), 500
|
||||
|
||||
@app.route('/add_app_config', methods=['POST'])
|
||||
def add_new_app_config():
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import multiprocessing
|
|||
workers = multiprocessing.cpu_count() * 2 + 1
|
||||
worker_class = 'geventwebsocket.gunicorn.workers.GeventWebSocketWorker'
|
||||
worker_connections = 1000
|
||||
timeout = 300
|
||||
timeout = 0 # Disable timeout completely
|
||||
|
||||
# Server socket
|
||||
bind = '0.0.0.0:7222'
|
||||
|
|
|
|||
|
|
@ -2640,6 +2640,8 @@
|
|||
document.getElementById('loadingOverlay').style.display = 'none';
|
||||
reconnectAttempts = 0;
|
||||
initializeUI();
|
||||
// Start sending heartbeats immediately after connection
|
||||
startHeartbeat();
|
||||
};
|
||||
|
||||
socket.onmessage = function(event) {
|
||||
|
|
@ -2648,7 +2650,8 @@
|
|||
console.log(`Data received from server:`, data);
|
||||
|
||||
if (data.type === 'heartbeat') {
|
||||
sendWebSocketMessage('heartbeat', {});
|
||||
// Reset heartbeat timeout on receiving heartbeat response
|
||||
resetHeartbeatTimeout();
|
||||
} else if (data.type === 'model_download_progress') {
|
||||
updateModelDownloadProgress(data.data);
|
||||
} else if (data.type === 'status_update') {
|
||||
|
|
@ -2675,14 +2678,70 @@
|
|||
};
|
||||
}
|
||||
|
||||
let heartbeatInterval;
|
||||
let heartbeatTimeout;
|
||||
|
||||
function startHeartbeat() {
|
||||
// Clear any existing intervals
|
||||
if (heartbeatInterval) {
|
||||
clearInterval(heartbeatInterval);
|
||||
}
|
||||
|
||||
// Send heartbeat every 60 seconds
|
||||
heartbeatInterval = setInterval(() => {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
socket.send(JSON.stringify({ type: 'heartbeat' }));
|
||||
// Set timeout to reconnect if no response within 10 seconds
|
||||
setHeartbeatTimeout();
|
||||
}
|
||||
}, 60000);
|
||||
}
|
||||
|
||||
function setHeartbeatTimeout() {
|
||||
if (heartbeatTimeout) {
|
||||
clearTimeout(heartbeatTimeout);
|
||||
}
|
||||
heartbeatTimeout = setTimeout(() => {
|
||||
console.log("Heartbeat timeout - attempting reconnect...");
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
handleReconnect();
|
||||
}, 65000); // Increased timeout to 65 seconds
|
||||
}
|
||||
|
||||
function resetHeartbeatTimeout() {
|
||||
if (heartbeatTimeout) {
|
||||
clearTimeout(heartbeatTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up on page unload
|
||||
window.addEventListener('beforeunload', () => {
|
||||
if (heartbeatInterval) {
|
||||
clearInterval(heartbeatInterval);
|
||||
}
|
||||
if (heartbeatTimeout) {
|
||||
clearTimeout(heartbeatTimeout);
|
||||
}
|
||||
if (socket) {
|
||||
socket.close();
|
||||
}
|
||||
});
|
||||
|
||||
function handleReconnect() {
|
||||
if (reconnectAttempts < maxReconnectAttempts) {
|
||||
reconnectAttempts++;
|
||||
document.getElementById('loadingOverlay').style.display = 'flex';
|
||||
document.getElementById('loadingMessage').textContent = `WebSocket disconnected. Attempting to reconnect (${reconnectAttempts}/${maxReconnectAttempts})...`;
|
||||
setTimeout(connectWebSocket, reconnectInterval);
|
||||
document.getElementById('loadingMessage').textContent =
|
||||
`WebSocket disconnected. Attempting to reconnect (${reconnectAttempts}/${maxReconnectAttempts})...`;
|
||||
|
||||
// Add exponential backoff
|
||||
const backoffTime = Math.min(1000 * Math.pow(2, reconnectAttempts), 30000);
|
||||
setTimeout(connectWebSocket, backoffTime);
|
||||
} else {
|
||||
document.getElementById('loadingMessage').textContent = 'Failed to connect to WebSocket. Please refresh the page or contact support.';
|
||||
document.getElementById('loadingMessage').textContent =
|
||||
'Failed to connect to WebSocket. Installation continues in background. Please refresh page to see status.';
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2873,9 +2932,14 @@
|
|||
setInterval(updateStatus, 5000);
|
||||
setInterval(updateLogs, 1000);
|
||||
|
||||
// Initialize model types
|
||||
refreshModelTypes();
|
||||
}
|
||||
|
||||
async function refreshModelTypes() {
|
||||
var data = {};
|
||||
data.cmd = 'refreshModelTypes';
|
||||
extendUIHelper(data); // lutzapps - initialize the available SHARED_MODEL_FOLDERS for the "Model Downloader" modelType select list
|
||||
await extendUIHelper(data); // lutzapps - initialize the available SHARED_MODEL_FOLDERS for the "Model Downloader" modelType select list
|
||||
}
|
||||
|
||||
// lutzapps - populate modeltype select options from shared_models
|
||||
|
|
@ -2933,53 +2997,49 @@
|
|||
case "refreshModelTypes":
|
||||
// refresh and optionally select the 'modelType' list for "Model Downloader"
|
||||
var modelTypeSelect = document.getElementById('modelType');
|
||||
|
||||
// get the data from the Server
|
||||
response = await fetch('/get_model_types');
|
||||
result = await response.json();
|
||||
|
||||
//alert(JSON.stringify(result)); // show the JSON-String
|
||||
var model_types = result; // get the JSON-Object
|
||||
var count = Object.keys(model_types).length; // count=18, when using the default SHARED_MODEL_FOLDERS dict
|
||||
|
||||
// the "/get_model_types" app.get_model_types_route() function checks
|
||||
// if the SHARED_MODELS_DIR shared files already exists at the "/workspace" location.
|
||||
// that only happens AFTER the the user clicked the "Create Shared Folders" button
|
||||
// on the "Settings" Tab of the app's WebUI.
|
||||
// it will return an empty model_types_dict, so the "Download Manager" does NOT get
|
||||
// the already in-memory SHARED_MODEL_FOLDERS code-generated default dict
|
||||
// BEFORE the workspace folders in SHARED_MODELS_DIR exists!
|
||||
//
|
||||
// when SHARED_MODELS_DIR exists (or updates), this function will be called via a Socket Message
|
||||
// to "refresh" its content automatically
|
||||
|
||||
var modelTypeSelected = modelTypeSelect.value; // remember the current selected modelType.option value
|
||||
modelTypeSelect.options.length = 0; // clear all current modelTypeSelect options
|
||||
|
||||
for (i = 0 ; i < count; i += 1) {
|
||||
modelTypeOption = document.createElement('option');
|
||||
|
||||
modelType = model_types[String(i)]['modelfolder'];
|
||||
modelTypeOption.setAttribute('value', modelType);
|
||||
modelTypeOption.appendChild(document.createTextNode(model_types[String(i)]['desc']));
|
||||
//if (modelFolder === modelTypeSelected) {
|
||||
// modelTypeOption.selected = true; // reselect it
|
||||
//}
|
||||
|
||||
modelTypeSelect.appendChild(modelTypeOption);
|
||||
if (!modelTypeSelect) {
|
||||
console.error('Model type select element not found');
|
||||
return;
|
||||
}
|
||||
|
||||
//modelTypeSelect.selectedIndex = modelfolder_index; // set the selected index
|
||||
//modelTypeSelect.options[mmodelfolder_index].selected = true; // and mark it as "selected" option
|
||||
if (modelTypeSelected === "") { // initial refresh, called by initializeUI() function
|
||||
modelTypeSelect.selectedIndex = 0; // use the first modelType option, usually "ckpt"
|
||||
MODELTYPE_SELECTED = modelTypeSelect.options[0].value; // NOT handled by the onchange() event handler
|
||||
}
|
||||
else {
|
||||
modelTypeSelect.value = modelTypeSelected; // (re-)apply the selected modelType option
|
||||
MODELTYPE_SELECTED = modelTypeSelected; // NOT handled by the onchange() event handler
|
||||
}
|
||||
try {
|
||||
// get the data from the Server
|
||||
response = await fetch('/get_model_types');
|
||||
result = await response.json();
|
||||
|
||||
var model_types = result; // get the JSON-Object
|
||||
var count = Object.keys(model_types).length;
|
||||
|
||||
var modelTypeSelected = modelTypeSelect.value; // remember the current selected modelType.option value
|
||||
modelTypeSelect.options.length = 0; // clear all current modelTypeSelect options
|
||||
|
||||
if (count === 0) {
|
||||
// Add a default option if no model types are available
|
||||
const defaultOption = document.createElement('option');
|
||||
defaultOption.text = 'Please create shared folders first';
|
||||
defaultOption.value = '';
|
||||
modelTypeSelect.add(defaultOption);
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < count; i++) {
|
||||
modelTypeOption = document.createElement('option');
|
||||
modelType = model_types[String(i)]['modelfolder'];
|
||||
modelTypeOption.setAttribute('value', modelType);
|
||||
modelTypeOption.appendChild(document.createTextNode(model_types[String(i)]['desc']));
|
||||
modelTypeSelect.appendChild(modelTypeOption);
|
||||
}
|
||||
|
||||
if (modelTypeSelected === "") {
|
||||
modelTypeSelect.selectedIndex = 0;
|
||||
MODELTYPE_SELECTED = modelTypeSelect.options[0].value;
|
||||
} else {
|
||||
modelTypeSelect.value = modelTypeSelected;
|
||||
MODELTYPE_SELECTED = modelTypeSelected;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error refreshing model types:', error);
|
||||
}
|
||||
break;
|
||||
|
||||
case "selectModelType":
|
||||
|
|
@ -3074,7 +3134,7 @@
|
|||
progressBar.style.width = '100%';
|
||||
progressBar.textContent = '100%';
|
||||
await loadModelFolders();
|
||||
showRecreateSymlinksButton();
|
||||
// Remove showRecreateSymlinksButton call since it's automatic now
|
||||
} else if (result.status === 'choice_required') {
|
||||
if (result.data.type === 'file') {
|
||||
showFileChoiceDialog(result.data, url, modelName, modelType, civitaiToken, hfToken);
|
||||
|
|
@ -3175,31 +3235,6 @@
|
|||
}
|
||||
}
|
||||
|
||||
function showRecreateSymlinksButton() {
|
||||
const buttonContainer = document.getElementById('recreate-symlinks-container');
|
||||
buttonContainer.innerHTML = `
|
||||
<button onclick="recreateSymlinks()" class="settings-button">Recreate Symlinks</button>
|
||||
<p id="symlink-status"></p>
|
||||
`;
|
||||
buttonContainer.style.display = 'block';
|
||||
}
|
||||
|
||||
async function recreateSymlinks() {
|
||||
const statusElement = document.getElementById('symlink-status');
|
||||
statusElement.textContent = 'Recreating symlinks...';
|
||||
try {
|
||||
const response = await fetch('/recreate_symlinks', { method: 'POST' });
|
||||
const data = await response.json();
|
||||
if (data.status === 'success') {
|
||||
statusElement.textContent = data.message;
|
||||
} else {
|
||||
statusElement.textContent = 'Error: ' + data.message;
|
||||
}
|
||||
} catch (error) {
|
||||
statusElement.textContent = 'Error: ' + error.message;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveCivitaiToken() {
|
||||
const token = document.getElementById('civitaiTokenSave').value;
|
||||
try {
|
||||
|
|
@ -3315,12 +3350,10 @@
|
|||
|
||||
// Call this function when the Models tab is opened
|
||||
document.querySelector('.navbar-tabs a[onclick="openTab(event, \'models-tab\')"]').addEventListener('click', function() {
|
||||
//alert("querySelector");
|
||||
loadModelFolders(); // lutzapps - this ModelFolders is NOT for the 'modelType' "select dropdown" model list
|
||||
extendUIHelper(); // lutzapps - select the last know MODELTYPE_SELECTED in the WebUI Dom Id 'modelType' "select dropdown" model list
|
||||
loadModelFolders();
|
||||
refreshModelTypes(); // Refresh model types when switching to Models tab
|
||||
loadCivitaiToken();
|
||||
loadHFToken(); // lutzapps - added HF_TOKEN ENV var Support
|
||||
|
||||
loadHFToken();
|
||||
updateExampleUrls();
|
||||
});
|
||||
|
||||
|
|
@ -3467,27 +3500,26 @@
|
|||
}
|
||||
}
|
||||
|
||||
async function updateFileBrowserStatus() {
|
||||
try {
|
||||
const response = await fetch('/filebrowser_status');
|
||||
const result = await response.json();
|
||||
const statusElement = document.getElementById('filebrowser-status');
|
||||
if (statusElement) {
|
||||
statusElement.textContent = result.status;
|
||||
}
|
||||
const startButton = document.getElementById('start-filebrowser');
|
||||
const stopButton = document.getElementById('stop-filebrowser');
|
||||
if (startButton && stopButton) {
|
||||
startButton.disabled = (result.status === 'running');
|
||||
stopButton.disabled = (result.status === 'stopped');
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error updating File Browser status:', error);
|
||||
}
|
||||
function updateFileBrowserStatus() {
|
||||
fetch('/filebrowser_status')
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
const statusElement = document.getElementById('filebrowser-status');
|
||||
if (statusElement) {
|
||||
statusElement.textContent = data.status;
|
||||
}
|
||||
const startButton = document.getElementById('start-filebrowser');
|
||||
const stopButton = document.getElementById('stop-filebrowser');
|
||||
if (startButton && stopButton) {
|
||||
startButton.disabled = (data.status === 'running');
|
||||
stopButton.disabled = (data.status === 'stopped');
|
||||
}
|
||||
})
|
||||
.catch(error => console.error('Error updating File Browser status:', error));
|
||||
}
|
||||
|
||||
// Call this function periodically to update the status
|
||||
setInterval(updateFileBrowserStatus, 5000);
|
||||
// Reduce the frequency of status updates
|
||||
setInterval(updateFileBrowserStatus, 30000); // Check every 30 seconds instead of 5 seconds
|
||||
|
||||
// Update the DOMContentLoaded event listener
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
|
|
@ -3542,7 +3574,8 @@
|
|||
console.log("WebSocket message received:", data);
|
||||
|
||||
if (data.type === 'heartbeat') {
|
||||
sendWebSocketMessage('heartbeat', {});
|
||||
// Reset heartbeat timeout on receiving heartbeat response
|
||||
resetHeartbeatTimeout();
|
||||
} else if (data.type === 'install_progress') {
|
||||
updateInstallProgress(data.data);
|
||||
} else if (data.type === 'install_log') {
|
||||
|
|
|
|||
|
|
@ -699,39 +699,12 @@ def download_and_unpack_venv_v2(app_name:str, app_configs:dict, send_websocket_m
|
|||
return False, error_message
|
||||
|
||||
send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 100, 'stage': 'Unpacking Complete'})
|
||||
send_websocket_message('install_log', {'app_name': app_name, 'log': 'Unpacking complete. Proceeding to clone repository...'})
|
||||
|
||||
### installing the App from GITHUB
|
||||
# Clone the repository if it doesn't exist
|
||||
success, message = clone_application(app_name)
|
||||
|
||||
print(f"'DEBUG_SETTINGS' after this run:\n{pretty_dict(DEBUG_SETTINGS)}")
|
||||
|
||||
### original "v1" code (very slow code because of STATISTICS glory
|
||||
|
||||
# unpack_command = f"tar -xzvf {downloaded_file} -C {venv_path}"
|
||||
# process = subprocess.Popen(unpack_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
|
||||
|
||||
# total_files = sum(1 for _ in subprocess.Popen(f"tar -tvf {downloaded_file}", shell=True, stdout=subprocess.PIPE).stdout)
|
||||
# files_processed = 0
|
||||
|
||||
# for line in process.stdout:
|
||||
# files_processed += 1
|
||||
# percentage = min(int((files_processed / total_files) * 100), 100)
|
||||
# send_websocket_message('install_progress', {
|
||||
# 'app_name': app_name,
|
||||
# 'percentage': percentage,
|
||||
# 'stage': 'Unpacking',
|
||||
# 'processed': files_processed,
|
||||
# 'total': total_files
|
||||
# })
|
||||
# send_websocket_message('install_log', {'app_name': app_name, 'log': f"Unpacking: {line.strip()}"})
|
||||
|
||||
# process.wait()
|
||||
# rc = process.returncode
|
||||
|
||||
### installing the App from GITHUB
|
||||
# Clone the repository if it doesn't exist
|
||||
success, error_message = clone_application(app_name, send_websocket_message)
|
||||
# Clone the repository
|
||||
success, message = clone_application(app_config, send_websocket_message)
|
||||
if not success:
|
||||
return False, message
|
||||
|
||||
# Clean up the downloaded file
|
||||
send_websocket_message('install_log', {'app_name': app_name, 'log': 'Cleaning up...'})
|
||||
|
|
@ -742,19 +715,10 @@ def download_and_unpack_venv_v2(app_name:str, app_configs:dict, send_websocket_m
|
|||
os.remove(downloaded_file)
|
||||
|
||||
send_websocket_message('install_log', {'app_name': app_name, 'log': 'Installation complete. Refresh page to start app'})
|
||||
save_install_status(app_name, 'completed', 100, 'Completed')
|
||||
send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."})
|
||||
return True, "Virtual environment installed successfully."
|
||||
|
||||
if success:
|
||||
save_install_status(app_name, 'completed', 100, 'Completed')
|
||||
send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."})
|
||||
return True, "Virtual environment installed successfully."
|
||||
else:
|
||||
return False, error_message
|
||||
|
||||
except requests.RequestException as e:
|
||||
error_message = f"Download/Decompression failed: {str(e)}"
|
||||
send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message})
|
||||
save_install_status(app_name, 'failed', 0, 'Failed')
|
||||
return False, error_message
|
||||
except Exception as e:
|
||||
error_message = f"Installation failed: {str(e)}\n{traceback.format_exc()}"
|
||||
save_install_status(app_name, 'failed', 0, 'Failed')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import subprocess
|
||||
import time
|
||||
import requests
|
||||
from requests.exceptions import Timeout
|
||||
|
||||
FILEBROWSER_PORT = 8181
|
||||
filebrowser_process = None
|
||||
|
|
@ -35,4 +37,10 @@ def stop_filebrowser():
|
|||
return False
|
||||
|
||||
def get_filebrowser_status():
|
||||
return 'running' if filebrowser_process and filebrowser_process.poll() is None else 'stopped'
|
||||
try:
|
||||
response = requests.get('http://localhost:7222/fileapp/', timeout=5)
|
||||
return 'running' if response.status_code == 200 else 'stopped'
|
||||
except Timeout:
|
||||
return 'timeout'
|
||||
except Exception:
|
||||
return 'unknown'
|
||||
|
|
|
|||
Loading…
Reference in a new issue