From 6a71890ae4f3e9256a7a52f83641e7d5908b2190 Mon Sep 17 00:00:00 2001 From: lutzapps Date: Sat, 9 Nov 2024 04:19:56 +0700 Subject: [PATCH] App Consolidation, app_configs and app_utils, Dockerfile --- .../better-ai-launcher/Dockerfile | 61 ++- .../better-ai-launcher/app/app.py | 176 +------ .../app/templates/index.html | 12 +- .../app/utils/app_configs.py | 433 ++++++++++++++---- .../better-ai-launcher/app/utils/app_utils.py | 409 +++++++++++------ .../app/utils/model_utils.py | 18 +- .../app/utils/shared_models.py | 14 +- 7 files changed, 683 insertions(+), 440 deletions(-) diff --git a/official-templates/better-ai-launcher/Dockerfile b/official-templates/better-ai-launcher/Dockerfile index ec31773..764a7b9 100644 --- a/official-templates/better-ai-launcher/Dockerfile +++ b/official-templates/better-ai-launcher/Dockerfile @@ -7,22 +7,75 @@ FROM ${BASE_IMAGE:-madiator2011/better-base:cuda12.4} AS base ARG BASE_IMAGE ENV BASE_IMAGE=$BASE_IMAGE +# lutzapps - replaced by above bake build-args #FROM madiator2011/better-base:cuda12.4 AS base # lutzapps - prepare for local developement and debugging # needed to change the ORDER of "apt-get commands" and move the "update-alternatives" for python3 # AFTER the "apt-get remove -y python3.10" cmd, OTHERWISE the symlink to python3 # is broken in the image and the VSCode debugger could not exec "python3" as CMD overwrite +# also fixed a boring "Blinker" blocking error -# Install Python 3.11, set it as default, and remove Python 3.10 RUN apt-get update && \ + ### ---> needed Tools for Installer # removed: 2x git nginx ffmpeg (as they are already installed with the base image) # added: pigz (for parallel execution of TAR files); zip (for easier folder compression) - apt-get install -y python3.11 python3.11-venv python3.11-dev python3.11-distutils \ - aria2 pigz zip pv rsync zstd libtcmalloc-minimal4 bc && \ - apt-get remove -y python3.10 python3.10-minimal libpython3.10-minimal libpython3.10-stdlib && \ + apt-get install -y aria2 pigz zip pv rsync zstd libtcmalloc-minimal4 bc \ + # add Python3.11 as system Python version, serving the Python Flask App + python3.11 python3.11-venv python3.11-dev python3.11-distutils && \ + # not remove Python3.10, as we need it for "official" app support (e.g. for kohya_ss VENV) + ###apt-get remove -y python3.10 python3.10-minimal libpython3.10-minimal libpython3.10-stdlib && \ + # + # setup an "alias" for "python" be symlinked to Python3.11 + # (which is the default anyway after this installation here of Python 3.11) update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \ + # + # setup the "python3" alias for Python3.11, as this is what the debugger needs (not work with 3.10) update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + # + # VENV will have their own "preferred/supported/recommended" Python version, as e.g. + # the "kohya_ss" app, only supports Python up to version 3.10 (but not 3.11) + # + # until here we have a broken "Blinker" installation from some base images before, + # and if we try to "update" "Blinker" for Python3.10 or via e.g. "pip install --upgrade blinker", + # or "pip install blinker==x.y.z.z", this breaks, as it was installed in an APT bundle, + # which can not be safely upgraded (= Uninstall/Reinstall)! It breaks as follows: + # we get a blinker 1.4 uninstall error chained by trying to uninstall "distutils": + # 8.568 Found existing installation: blinker 1.4 + # 8.782 error: uninstall-distutils-installed-package + # 8.782 + # 8.782 × Cannot uninstall blinker 1.4 + # 8.782 ╰─> It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall. + # that not only blocks building the docker image, but later also breaks the "kohya_ss" app during setup, + # which try to install Blinker and Python310-venv from "setup-runpod.sh": + # # Install tk and python3.10-venv + # echo "Installing tk and python3.10-venv..." + # apt update -y && apt install -y python3-tk python3.10-venv + # + # Python 3.10 needs to run as Kohya's "official" requirement, and is used in the VENV + # + # first uninstall the APT bundle package for "Blinker" + apt-get remove -y python3-blinker && \ + # then re-install the APT unbundled package of "Blinker back", + # together with Python3.10 venv, which we need to setup the kohya_ss VENV + apt-get install -y python3-tk python3.10-venv && \ + # this re-captures back the "python3" alias for Python3.10, but not the "python" alias (stays for Python3.11) + # the "Python3.11" and "Python3.10" cmds work too + # global PIP is 3.11, VENV pip is 3.10 + # + # ---> CUDA 12.4 Toolkit (is already in the 12.4 base-image) + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb && \ + ### need to refresh the repository metatdata, after downloading this NVIDIA downloaded package list!!! + apt-get update && \ + apt-get -y install cuda-toolkit-12-4 && \ + # + # ---> get the latest cuDNN 9.x version supporting CUDA 12.x + # remove the current dev package which depends on libcudnn9-cuda-12 and breaks the upgrade otherwise + apt-get remove -y --allow-change-held-packages libcudnn9-cuda-12 libcudnn9-dev-cuda-12 && \ + apt-get -y --allow-change-held-packages install cudnn-cuda-12 && \ + # + # clean-up resources and caches apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/official-templates/better-ai-launcher/app/app.py b/official-templates/better-ai-launcher/app/app.py index 7021ad8..871e8f0 100644 --- a/official-templates/better-ai-launcher/app/app.py +++ b/official-templates/better-ai-launcher/app/app.py @@ -28,8 +28,7 @@ from utils.shared_models import ( SHARED_MODELS_DIR, SHARED_MODEL_FOLDERS, SHARED_MODEL_FOLDERS_FILE, ensure_shared_models_folders, APP_INSTALL_DIRS, APP_INSTALL_DIRS_FILE, init_app_install_dirs, # APP_INSTALL_DIRS dict/file/function MAP_APPS, sync_with_app_configs_install_dirs, # internal MAP_APPS dict and sync function - SHARED_MODEL_APP_MAP, SHARED_MODEL_APP_MAP_FILE, init_shared_model_app_map, # SHARED_MODEL_APP_MAP dict/file/function - write_dict_to_jsonfile, read_dict_from_jsonfile, PrettyDICT # JSON helper functions + SHARED_MODEL_APP_MAP, SHARED_MODEL_APP_MAP_FILE, init_shared_model_app_map # SHARED_MODEL_APP_MAP dict/file/function ) # the "update_model_symlinks()" function replaces the app.py function with the same same # and redirects to same function name "update_model_symlinks()" in the new "utils.shared_models" module @@ -340,92 +339,12 @@ def remove_existing_app_config(app_name): return jsonify({'status': 'success', 'message': f'App {app_name} removed successfully'}) return jsonify({'status': 'error', 'message': f'App {app_name} not found'}) -# unused function -def obsolate_update_model_symlinks(): - # lutzapps - CHANGE #3 - use the new "shared_models" module for app model sharing - # remove this whole now unused function - return "replaced by utils.shared_models.update_model_symlinks()" - # modified function def setup_shared_models(): # lutzapps - CHANGE #4 - use the new "shared_models" module for app model sharing jsonResult = update_model_symlinks() return SHARED_MODELS_DIR # shared_models_dir is now owned and managed by the "shared_models" utils module - # remove below unused code - - shared_models_dir = '/workspace/shared_models' - model_types = ['Stable-diffusion', 'VAE', 'Lora', 'ESRGAN'] - - # Create shared models directory if it doesn't exist - os.makedirs(shared_models_dir, exist_ok=True) - - for model_type in model_types: - shared_model_path = os.path.join(shared_models_dir, model_type) - - # Create shared model type directory if it doesn't exist - os.makedirs(shared_model_path, exist_ok=True) - - # Create a README file in the shared models directory - readme_path = os.path.join(shared_models_dir, 'README.txt') - if not os.path.exists(readme_path): - with open(readme_path, 'w') as f: - f.write("Upload your models to the appropriate folders:\n\n") - f.write("- Stable-diffusion: for Stable Diffusion models\n") - f.write("- VAE: for VAE models\n") - f.write("- Lora: for LoRA models\n") - f.write("- ESRGAN: for ESRGAN upscaling models\n\n") - f.write("These models will be automatically linked to all supported apps.") - - print(f"Shared models directory created at {shared_models_dir}") - print("Shared models setup completed.") - - return shared_models_dir - -# unused function -def obsolate_update_model_symlinks(): - # lutzapps - CHANGE #5 - use the new "shared_models" module for app model sharing - # remove this whole now unused function - return "replaced by utils.shared_models.update_model_symlinks()" - - shared_models_dir = '/workspace/shared_models' - apps = { - 'stable-diffusion-webui': '/workspace/stable-diffusion-webui/models', - 'stable-diffusion-webui-forge': '/workspace/stable-diffusion-webui-forge/models', - 'ComfyUI': '/workspace/ComfyUI/models' - } - model_types = ['Stable-diffusion', 'VAE', 'Lora', 'ESRGAN'] - - for model_type in model_types: - shared_model_path = os.path.join(shared_models_dir, model_type) - - if not os.path.exists(shared_model_path): - continue - - for app, app_models_dir in apps.items(): - if app == 'ComfyUI': - if model_type == 'Stable-diffusion': - app_model_path = os.path.join(app_models_dir, 'checkpoints') - elif model_type == 'Lora': - app_model_path = os.path.join(app_models_dir, 'loras') - elif model_type == 'ESRGAN': - app_model_path = os.path.join(app_models_dir, 'upscale_models') - else: - app_model_path = os.path.join(app_models_dir, model_type.lower()) - else: - app_model_path = os.path.join(app_models_dir, model_type) - - # Create the app model directory if it doesn't exist - os.makedirs(app_model_path, exist_ok=True) - - # Create symlinks for each file in the shared model directory - for filename in os.listdir(shared_model_path): - src = os.path.join(shared_model_path, filename) - dst = os.path.join(app_model_path, filename) - if os.path.isfile(src) and not os.path.exists(dst): - os.symlink(src, dst) - - print("Model symlinks updated.") def update_symlinks_periodically(): while True: @@ -436,57 +355,6 @@ def start_symlink_update_thread(): thread = threading.Thread(target=update_symlinks_periodically, daemon=True) thread.start() -# unused function -def obsolate_recreate_symlinks(): - # lutzapps - CHANGE #6 - use the new "shared_models" module for app model sharing - # remove this whole now unused function - return "replaced by utils.shared_models.update_model_symlinks()" - - shared_models_dir = '/workspace/shared_models' - apps = { - 'stable-diffusion-webui': '/workspace/stable-diffusion-webui/models', - 'stable-diffusion-webui-forge': '/workspace/stable-diffusion-webui-forge/models', - 'ComfyUI': '/workspace/ComfyUI/models' - } - model_types = ['Stable-diffusion', 'VAE', 'Lora', 'ESRGAN'] - - for model_type in model_types: - shared_model_path = os.path.join(shared_models_dir, model_type) - - if not os.path.exists(shared_model_path): - continue - - for app, app_models_dir in apps.items(): - if app == 'ComfyUI': - if model_type == 'Stable-diffusion': - app_model_path = os.path.join(app_models_dir, 'checkpoints') - elif model_type == 'Lora': - app_model_path = os.path.join(app_models_dir, 'loras') - elif model_type == 'ESRGAN': - app_model_path = os.path.join(app_models_dir, 'upscale_models') - else: - app_model_path = os.path.join(app_models_dir, model_type.lower()) - else: - app_model_path = os.path.join(app_models_dir, model_type) - - # Remove existing symlinks - if os.path.islink(app_model_path): - os.unlink(app_model_path) - elif os.path.isdir(app_model_path): - shutil.rmtree(app_model_path) - - # Create the app model directory if it doesn't exist - os.makedirs(app_model_path, exist_ok=True) - - # Create symlinks for each file in the shared model directory - for filename in os.listdir(shared_model_path): - src = os.path.join(shared_model_path, filename) - dst = os.path.join(app_model_path, filename) - if os.path.isfile(src) and not os.path.exists(dst): - os.symlink(src, dst) - - return "Symlinks recreated successfully." - # modified function @app.route('/recreate_symlinks', methods=['POST']) def recreate_symlinks_route(): @@ -494,13 +362,6 @@ def recreate_symlinks_route(): jsonResult = update_model_symlinks() return jsonResult - # remove below unused code - - try: - message = recreate_symlinks() - return jsonify({'status': 'success', 'message': message}) - except Exception as e: - return jsonify({'status': 'error', 'message': str(e)}) # modified function @app.route('/create_shared_folders', methods=['POST']) @@ -508,39 +369,6 @@ def create_shared_folders(): # lutzapps - CHANGE #8 - use the new "shared_models" module for app model sharing jsonResult = ensure_shared_models_folders() return jsonResult - # remove below unused code - - try: - shared_models_dir = '/workspace/shared_models' - model_types = ['Stable-diffusion', 'Lora', 'embeddings', 'VAE', 'hypernetworks', 'aesthetic_embeddings', 'controlnet', 'ESRGAN'] - - # Create shared models directory if it doesn't exist - os.makedirs(shared_models_dir, exist_ok=True) - - for model_type in model_types: - shared_model_path = os.path.join(shared_models_dir, model_type) - - # Create shared model type directory if it doesn't exist - os.makedirs(shared_model_path, exist_ok=True) - - # Create a README file in the shared models directory - readme_path = os.path.join(shared_models_dir, 'README.txt') - if not os.path.exists(readme_path): - with open(readme_path, 'w') as f: - f.write("Upload your models to the appropriate folders:\n\n") - f.write("- Stable-diffusion: for Stable Diffusion checkpoints\n") - f.write("- Lora: for LoRA models\n") - f.write("- embeddings: for Textual Inversion embeddings\n") - f.write("- VAE: for VAE models\n") - f.write("- hypernetworks: for Hypernetwork models\n") - f.write("- aesthetic_embeddings: for Aesthetic Gradient embeddings\n") - f.write("- controlnet: for ControlNet models\n") - f.write("- ESRGAN: for ESRGAN upscaling models\n\n") - f.write("These models will be automatically linked to all supported apps.") - - return jsonify({'status': 'success', 'message': 'Shared model folders created successfully.'}) - except Exception as e: - return jsonify({'status': 'error', 'message': str(e)}) def save_civitai_token(token): with open(CIVITAI_TOKEN_FILE, 'w') as f: @@ -644,7 +472,7 @@ def get_model_types_route(): 'desc': model_type_description } - i = i + 1 + i += 1 return model_types_dict diff --git a/official-templates/better-ai-launcher/app/templates/index.html b/official-templates/better-ai-launcher/app/templates/index.html index 0d2408c..4dae966 100644 --- a/official-templates/better-ai-launcher/app/templates/index.html +++ b/official-templates/better-ai-launcher/app/templates/index.html @@ -2418,7 +2418,7 @@
-
4 +
@@ -2838,6 +2838,7 @@ function initializeUI() { updateStatus(); + // TODO: need a way this 2 functions not pollute the logs every second or 5 seconds setInterval(updateStatus, 5000); setInterval(updateLogs, 1000); @@ -3281,15 +3282,6 @@ }); } - // lutzapps - obsolete function (can be deleted) - function copyToClipboard(text) { - navigator.clipboard.writeText(text).then(() => { - alert('URL copied to clipboard!'); - }, (err) => { - console.error('Could not copy text: ', err); - }); - } - // Call this function when the Models tab is opened document.querySelector('.navbar-tabs a[onclick="openTab(event, \'models-tab\')"]').addEventListener('click', function() { //alert("querySelector"); diff --git a/official-templates/better-ai-launcher/app/utils/app_configs.py b/official-templates/better-ai-launcher/app/utils/app_configs.py index f79ed98..47a1507 100644 --- a/official-templates/better-ai-launcher/app/utils/app_configs.py +++ b/official-templates/better-ai-launcher/app/utils/app_configs.py @@ -3,71 +3,251 @@ import xml.etree.ElementTree as ET import requests import json -def fetch_app_info(): - url = "https://better.s3.madiator.com/" - response = requests.get(url) - root = ET.fromstring(response.content) - - app_info = {} - for content in root.findall('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'): - key = content.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text - size = int(content.find('{http://s3.amazonaws.com/doc/2006-03-01/}Size').text) - app_name = key.split('/')[0] - - # lutzapps - fix "bug" in key element of the S3 XML document - # all other three apps have a "key" element like "bcomfy/bcomfy.tar.gz" or "bforge/bforge.tar.gz", - # with their "app_name" prefix + "/" + tar_filename - # only kohya is missing this "app_name" prefix and has a key element of only its tar_filename "bkohya.tar.gz" - # this results in the app_name "bkohya.tar.gz", instead of only "bkohya" - # TODO for madiator - move the "bkohya.tar.gz" into a subfolder "bkohya" in your S3 bucket - app_name = app_name.replace(".tar.gz", "") # cut any extension postfixes resulting from the wrong key.split() command - - if app_name in ['ba1111', 'bcomfy', 'bforge', 'bkohya']: # lutzapps - added new kohya app - app_info[app_name] = { - 'download_url': f"https://better.s3.madiator.com/{key}", - 'size': size - } - - return app_info +# this is the replacement for the XML manifest, and defines all app_configs in full detail +APP_CONFIGS_MANIFEST_URL = "https://better.s3.madiator.com/app_configs.json" +# if this JSON can not be downloaded, the below code defaults apply +# this app_configs dict can also be generated from code when at least one of following +# 2 ENV vars are found with following values: +# 1. LOCAL_DEBUG = 'True' # this ENV var should not be passed when in the RUNPOD environment, as it disabled the CF proxy Urls of the App-Manager +# and this ENV var also controls some other aspects of the app. +# +# 2. APP_CONFIGS_FILE = 'True' # only exists for this one purpose, to generate the below Dict as file +# "/workspace/_app_configs.json", which then can be uploaded to the above defined APP_CONFIGS_MANIFEST_URL +# NOTE: app_configs = { 'bcomfy': { + 'id': 'bcomfy', 'name': 'Better Comfy UI', 'command': 'cd /workspace/bcomfy && . ./bin/activate && cd /workspace/ComfyUI && python main.py --listen --port 3000 --enable-cors-header', 'venv_path': '/workspace/bcomfy', 'app_path': '/workspace/ComfyUI', 'port': 3000, + 'download_url': 'https://better.s3.madiator.com/bcomfy/bcomfy.tar.gz', # (2024-11-08 18:50:00Z - lutzapps) + #'venv_uncompressed_size': 6452737952, # uncompressed size of the tar-file (in bytes) - lutzapps new version + 'venv_uncompressed_size': 6155295493, # uncompressed size of the tar-file (in bytes) - original version + #'archive_size': 3389131462 # tar filesize (in bytes) - lutzapps new version + 'archive_size': 3179595118, # tar filesize (in bytes) - original version + #'sha256_hash': '18e7d71b75656924f98d5b7fa583aa7c81425f666a703ef85f7dd0acf8f60886', # lutzapps new version + 'sha256_hash': '7fd60808a120a1dd05287c2a9b3d38b3bdece84f085abc156e0a2ee8e6254b84', # original version + 'repo_url': 'https://github.com/comfyanonymous/ComfyUI.git', + 'branch_name': '', # empty branch_name means default = 'master' + 'commit': '', # or commit hash (NYI) + 'recursive': False, + 'refresh': False, + 'custom_nodes': [ # following custom_nodes will be git cloned and installed with "pip install -r requirements.txt" (in Testing) + { + 'name': 'ComfyUI-Manager (ltdrdata)', # this node is installed in the VENV + 'path': 'ComfyUI-Manager', + 'repo_url': 'https://github.com/ltdrdata/ComfyUI-Manager.git' + }, + { + 'name': 'ComfyUI-Essentials (cubic)', # this node is installed in the VENV + 'path': 'ComfyUI_essentials', + 'repo_url': 'https://github.com/cubiq/ComfyUI_essentials' + } + ### planned custom nodes - To Be Discussed + # { + # 'name': 'rgthree comfy', + # 'path': 'rgthree-comfy', + # 'repo_url': 'https://github.com/rgthree/rgthree-comfy' + # }, + # { + # 'name': 'was node suite comfyui', + # 'path': 'was-node-suite-comfyui', + # 'repo_url': 'https://github.com/WASasquatch/was-node-suite-comfyui' + # }, + # { + # 'name': 'comfyui controlnet aux', + # 'path': 'comfyui_controlnet_aux', + # 'repo_url': 'https://github.com/Fannovel16/comfyui_controlnet_aux' + # }, + # { + # 'name': 'x-flux-comfyui (XLabs-AI)', + # 'path': 'x-flux-comfyui', + # 'repo_url': 'https://github.com/XLabs-AI/x-flux-comfyui' + # }, + # { + # 'name': 'ComfyUI-GGUF (city96)', + # 'path': 'ComfyUI-GGUF', + # 'repo_url': 'https://github.com/city96/ComfyUI-GGUF' + # }, + # { + # 'name': 'ComfyUI-Florence2 (kijai)', + # 'path': 'ComfyUI-Florence2F', + # 'repo_url': 'https://github.com/kijai/ComfyUI-Florence2' + # }, + # { + # 'name': 'ComfyUI-KJNodes (kijai)', + # 'path': 'ComfyUI-KJNodes', + # 'repo_url': 'https://github.com/kijai/ComfyUI-KJNodes' + # }, + # { + # 'name': 'ComfyUI_UltimateSDUpscale (ssitu)', + # 'path': 'ComfyUI_UltimateSDUpscale', + # 'repo_url': 'https://github.com/ssitu/ComfyUI_UltimateSDUpscale' + # }, + # { + # 'name': 'ControlAltAI Nodes (gseth)', + # 'path': 'ControlAltAI-Nodes', + # 'repo_url': 'https://github.com/gseth/ControlAltAI-Nodes' + # }, + # { + # 'name': 'ComfyUI Easy-Use (yolain)', + # 'path': 'ComfyUI-Easy-Use', + # 'repo_url': 'https://github.com/yolain/ComfyUI-Easy-Use' + # }, + # { + # 'name': 'ComfyUI Impact-Pack (tdrdata)', + # 'path': 'ComfyUI-Impact-Pack', + # 'repo_url': 'https://github.com/ltdrdata/ComfyUI-Impact-Pack' + # } + ] }, 'bforge': { + 'id': 'bforge', # app_name 'name': 'Better Forge', 'command': 'cd /workspace/bforge && . ./bin/activate && cd /workspace/stable-diffusion-webui-forge && ./webui.sh -f --listen --enable-insecure-extension-access --api --port 7862', 'venv_path': '/workspace/bforge', 'app_path': '/workspace/stable-diffusion-webui-forge', 'port': 7862, + 'download_url': 'https://better.s3.madiator.com/bforge/bforge.tar.gz', + 'venv_uncompressed_size': 7689838771, # uncompressed size of the tar-file (in bytes), + 'archive_size': 3691004078, # tar filesize (in bytes) + 'sha256_hash': 'e87dae2324a065944c8d36d6ac4310af6d2ba6394f858ff04a34c51aa5f70bfb', + 'repo_url': 'https://github.com/lllyasviel/stable-diffusion-webui-forge.git', + 'branch_name': '', # empty branch_name means default = 'master' + 'commit': '', # or commit hash (NYI) + 'clone_recursive': False, + 'refresh': False }, 'ba1111': { + 'id': 'ba1111', # app_name 'name': 'Better A1111', 'command': 'cd /workspace/ba1111 && . ./bin/activate && cd /workspace/stable-diffusion-webui && ./webui.sh -f --listen --enable-insecure-extension-access --api --port 7863', 'venv_path': '/workspace/ba1111', 'app_path': '/workspace/stable-diffusion-webui', 'port': 7863, + 'download_url': 'https://better.s3.madiator.com/ba1111/ba1111.tar.gz', + 'venv_uncompressed_size': 6794367826, # uncompressed size of the tar-file (in bytes), + 'archive_size': 3383946179, # tar filesize (in bytes) + 'sha256_hash': '1d70276bc93f5f992a2e722e76a469bf6a581488fa1723d6d40739f3d418ada9', + 'repo_url': 'https://github.com/AUTOMATIC1111/stable-diffusion-webui.git', + 'branch_name': '', # empty branch_name means default = 'master' + 'commit': '', # or commit hash (NYI) + 'clone_recursive': False, + 'refresh': False }, 'bkohya': { + 'id': 'bkohya', # app_name 'name': 'Better Kohya', - 'command': 'cd /workspace/bkohya && . ./bin/activate && cd /workspace/kohya_ss && ./gui.sh --listen --port 7860', + 'command': 'cd /workspace/bkohya && . ./bin/activate && cd /workspace/kohya_ss && python ./kohya_gui.py --headless --share --server_port 7860', # TODO!! check ./kohya_gui.py + ### for Gradio supported reverse proxy: + # --share -> Share the gradio UI + # --root_path ROOT_PATH -> root_path` for Gradio to enable reverse proxy support. e.g. /kohya_ss + # --listen LISTEN -> IP to listen on for connections to Gradio + + # usage: kohya_gui.py [-h] [--config CONFIG] [--debug] [--listen LISTEN] + # [--username USERNAME] [--password PASSWORD] + # [--server_port SERVER_PORT] [--inbrowser] [--share] + # [--headless] [--language LANGUAGE] [--use-ipex] + # [--use-rocm] [--do_not_use_shell] [--do_not_share] + # [--requirements REQUIREMENTS] [--root_path ROOT_PATH] + # [--noverify] + # + # options: + # -h, --help show this help message and exit + # --config CONFIG Path to the toml config file for interface defaults + # --debug Debug on + # --listen LISTEN IP to listen on for connections to Gradio + # --username USERNAME Username for authentication + # --password PASSWORD Password for authentication + # --server_port SERVER_PORT + # Port to run the server listener on + # --inbrowser Open in browser + # --share Share the gradio UI + # --headless Is the server headless + # --language LANGUAGE Set custom language + # --use-ipex Use IPEX environment + # --use-rocm Use ROCm environment + # --do_not_use_shell Enforce not to use shell=True when running external + # commands + # --do_not_share Do not share the gradio UI + # --requirements REQUIREMENTS + # requirements file to use for validation + # --root_path ROOT_PATH + # `root_path` for Gradio to enable reverse proxy + # support. e.g. /kohya_ss + # --noverify Disable requirements verification + 'venv_path': '/workspace/bkohya', 'app_path': '/workspace/kohya_ss', 'port': 7860, + 'download_url': 'https://better.s3.madiator.com/kohya.tar.gz', # (2024-11-08 13:13:00Z) - lutzapps + 'venv_uncompressed_size': 12128345264, # uncompressed size of the tar-file (in bytes) + 'archive_size': 6314758227, # tar filesize (in bytes) + 'sha256_hash': '9a0c0ed5925109e82973d55e28f4914fff6728cfb7f7f028a62e2ec1a9e4f60a', + 'repo_url': 'https://github.com/bmaltais/kohya_ss.git', + 'branch_name': 'sd3-flux.1', # make sure we use Kohya with FLUX support branch + # this branch also uses a 'sd-scripts' HEAD branch of 'SD3', which gets automatically checked-out too + 'commit': '', # or commit hash (NYI) + 'clone_recursive': True, # is recursive clone + 'refresh': False } } -def update_app_configs(): +# lutzapps - not used anymore TODO: remove later +""" def fetch_app_info(): + manifest_url = "https://better.s3.madiator.com/" + download_base_url = "https://better.s3.madiator.com/" # could be different base as the manifest file + + app_info = {} + + try: # be graceful when the server is not reachable, be it S3 or anything else + response = requests.get(manifest_url) + root = ET.fromstring(response.content) + + for content in root.findall('{http://s3.amazonaws.com/doc/2006-03-01/}Contents'): + app_name_and_url = content.find('{http://s3.amazonaws.com/doc/2006-03-01/}Key').text + + app_name = app_name_and_url.split('/')[0] # e.g. "bkohya/bkohya.tar.gz" -> "bkohya" + download_url = os.path.join(download_base_url, app_name_and_url) + + if not (app_name in ['ba1111', 'bcomfy', 'bforge', 'bkohya']): + continue # skip unsupported app + + # load code defaults + archive_size = app_configs[app_name]["archive_size"] + venv_uncompressed_size = app_configs[app_name]["venv_uncompressed_size"] + sha256_hash = app_configs[app_name]["sha256_hash"] + + try: # try to find overwrites from code defaults + archive_size = int(content.find('archive_size').text) + venv_uncompressed_size = int(content.find('{http://s3.amazonaws.com/doc/2006-03-01/}venv_uncompressed_size').text) + sha256_hash = int(content.find('{http://s3.amazonaws.com/doc/2006-03-01/}sha256_hash').text) + except: # swallow any exception, mainly from not being defined (yet) in the XML manifest + print(f"App '{app_name}' Metadata could not be found in manifest '{manifest_url}', using code defaults!") + + app_info[app_name] = { + 'download_url': download_url, + 'archive_size': archive_size, + 'venv_uncompressed_size': venv_uncompressed_size, # TODO: provide in XML manifest + 'sha256_hash': sha256_hash # TODO: provide in XML manifest + } + + except requests.RequestException as e: # server not reachable, return empty dict + print(f"Manifest Url '{manifest_url}' not reachable, using code defaults!") + + return app_info + """ +# lutzapps - not used anymore TODO: remove later +""" def update_app_configs(): app_info = fetch_app_info() for app_name, info in app_info.items(): if app_name in app_configs: - app_configs[app_name].update(info) + app_configs[app_name].update(info) """ -def get_app_configs(): +def get_app_configs() -> dict: return app_configs def add_app_config(app_name, config): @@ -78,7 +258,8 @@ def remove_app_config(app_name): del app_configs[app_name] # Update app_configs when this module is imported -update_app_configs() +# lutzapps - not used anymore TODO: remove later +#update_app_configs() ### lutzapps section @@ -108,7 +289,7 @@ def write_dict_to_jsonfile(dict:dict, json_filepath:str, overwrite:bool=False) - return True, "" # success # helper function called by init_app_install_dirs(), init_shared_model_app_map(), init_shared_models_folders() and init_debug_settings() -def read_dict_from_jsonfile(json_filepath:str) -> dict: +def read_dict_from_jsonfile(json_filepath:str) -> tuple [dict, str]: # Read JSON file from 'json_filepath' and return it as 'dict' try: @@ -135,9 +316,9 @@ def pretty_dict(dict:dict) -> str: return dict_string -# helper function for "init_app_install_dirs(), "init_shared_model_app_map()", "init_shared_models_folders()" and "inir_debug_settings()" -def init_global_dict_from_file(dict:dict, dict_filepath:str, dict_description:str, SHARED_MODELS_DIR:str="") -> bool: - # load or initialize the 'dict' for 'dict_description' from 'dict_filepath' +# helper function for "init_app_install_dirs(), "init_shared_model_app_map()", "init_shared_models_folders()" and "inir_DEBUG_SETTINGS()" +def load_global_dict_from_file(dict:dict, dict_filepath:str, dict_description:str, SHARED_MODELS_DIR:str="", write_file:bool=True) -> tuple[bool, dict]: + # returns the 'dict' for 'dict_description' from 'dict_filepath' try: if not SHARED_MODELS_DIR == "" and not os.path.exists(SHARED_MODELS_DIR): @@ -170,51 +351,40 @@ def init_global_dict_from_file(dict:dict, dict_filepath:str, dict_description:st print(f"\nUsing {'external' if dict_filepath_found else 'default'} '{dict_description}':\n{pretty_dict(dict)}") except Exception as e: - error_msg = f"ERROR in shared_models:init_global_dict_from_file() - initializing dict Map File '{dict_filepath}'\nException: {str(e)}" - print(error_msg) + print(f"ERROR in shared_models:load_global_dict_from_file() - initializing dict Map File '{dict_filepath}'\nException: {str(e)}") - return False, error_msg + return False, {} - return True, "" # success + return True, dict # success DEBUG_SETTINGS_FILE = "/workspace/_debug_settings.json" DEBUG_SETTINGS = { # these setting will be READ: - "manifests": { # uncompressed sizes of the tar-files - "bcomfy": { - "venv_uncompressed_size": 6155283197, - "sha256_hash": "" - }, - "ba1111": { - "venv_uncompressed_size": 6794355530, - "sha256_hash": "" - }, - "bforge": { - "venv_uncompressed_size": 7689838771, - "sha256_hash": "" - }, - "bkohya": { - "venv_uncompressed_size": 12192767148, - "sha256_hash": "" - } - }, - "installer_codeversion": "2", # can be "1" (original) or "2" (fast) - "delete_tarfile_after_download": "1", # can be set to "0" to test only local unpack time and github setup - "use_bkohya_tar_folder_fix": "1", # the fix unpacks to "/workspace" and not to "/workspace/bkohya" - "use_bkohya_local_venv_symlink": "1", # when active, creates a folder symlink "venv" in "/workspace/kohya_ss" -> "/workspace/bkohya" VENV + "APP_CONFIGS_MANIFEST_URL": "", # this setting, when not blank, overwrites the global APP_CONFIGS_MANIFEST_URL + "installer_codeversion": "v2", # can be "v1" (original) or "v2" (fast) + "delete_tar_file_after_download": True, # can be set to True to test only local unpack time and github setup + "create_bkohya_to_local_venv_symlink": True, # when True, creates a folder symlink "venv" in "/workspace/kohya_ss" -> "/workspace/bkohya" VENV + "skip_to_github_stage": False, # when True, skip download and decompression stage and go directly to GH repo installation # these settings will be WRITTEN: - "used_local_tar": "0", # works together with the above TAR local caching - "app_name": "", - "tar_filename": "", - "download_url": "", - "total_duration_download": "0", - "total_duration_unpack": "0", - "total_duration": "0" + "app_name": "", # last app_name the code run on + "used_local_tarfile": True, # works together with the above TAR local caching + "tar_filename": "", # last local tar_filename used + "download_url": "", # last used tar download_url + "total_duration_download": "00:00:00", # timespan-str "hh:mm:ss" + "total_duration_unpack": "00:00:00", # timespan-str "hh:mm:ss" + "total_duration": "00:00:00" # timespan-str "hh:mm:ss" } def init_debug_settings(): global DEBUG_SETTINGS - init_global_dict_from_file(DEBUG_SETTINGS, DEBUG_SETTINGS_FILE, "DEBUG_SETTINGS") + + local_debug = os.environ.get('LOCAL_DEBUG', 'False') # support local browsing for development/debugging + generate_debug_settings_file = os.environ.get('DEBUG_SETTINGS_FILE', 'False') # generate the DEBUG_SETTINGS_FILE, if not exist already + write_file_if_not_exist = local_debug == 'True' or generate_debug_settings_file == 'True' + + success, dict = load_global_dict_from_file(DEBUG_SETTINGS, DEBUG_SETTINGS_FILE, "DEBUG_SETTINGS", write_file=write_file_if_not_exist) + if success: + DEBUG_SETTINGS = dict # read from DEBUG_SETTINGS # installer_codeversion = DEBUG_SETTINGS['installer_codeversion'] # read from DEBUG_SETTINGS @@ -233,14 +403,60 @@ def write_debug_setting(setting_name:str, setting_value:str): # lutzapps - init some settings from DEBUG_SETTINGS_FILE init_debug_settings() -# lutzapps - add kohya_ss support and required local VENV +APP_CONFIGS_FILE = APP_CONFIGS_MANIFEST_URL # default is the online manifest url defined as "master" +# can be overwritten with DEBUG_SETTINGS['APP_CONFIGS_MANIFEST_URL'], e.g. point to "/workspace/_app_configs.json" +# # which is the file, that is generated when the ENV var LOCAL_DEBUG='True' or the ENV var APP_CONFIGS_FILE='True' +# NOTE: an existing serialized dict in the "/workspace" folder will never be overwritten agin from the code defaults, +# and "wins" against the code-defaults. So even changes in the source-code for this dicts will NOT be used, +# when a local file exists. The idea here is that it is possible to overwrite code-defaults. +# BUT as long as the APP_CONFIGS_MANIFEST_URL not gets overwritten, the global "app_configs" dict will be always loaded +# from the central S3 server, or whatever is defined. +# the only way to overwrite this url, is via the DEBUG_SETTINGS_FILE "/workspace/_debug_settings.json" +# the default source-code setting for DEBUG_SETTINGS['APP_CONFIGS_MANIFEST_URL']: "" (is an empty string), +# which still makes the default APP_CONFIGS_MANIFEST_URL the central master. +# only when this setting is not empty, it can win against the central url, but also only when the Url is valid (locally or remote) +# should there be an invalid Url (central or local), or any other problem, then the code-defaults will be used. +# +# The DEBUG_SETTINGS_FILE is a dict which helps during debugging, testing of APP Installations, +# and generating ENV TAR files. +# Is will also NOT be generated as external FILE, as long the same 2 ENV vars, which control the APP_CONFIGS_FILE generation are set: +# LOCAL_DEBUG='True' or APP_CONFIGS_FILE='True' +# +# SUMMARY: The DEBUG_SETTINGS and APP_CONFIGS (aka app_configs in code) will never be written to the /workspace, +# when the IMAGE is used normally. + +def init_app_configs(): + global APP_CONFIGS_FILE + global app_configs + + # check for overwrite of APP_CONFIGS_MANIFEST_URL + if not DEBUG_SETTINGS['APP_CONFIGS_MANIFEST_URL'] == "": + APP_CONFIGS_FILE = DEBUG_SETTINGS['APP_CONFIGS_MANIFEST_URL'] + + local_debug = os.environ.get('LOCAL_DEBUG', 'False') # support local browsing for development/debugging + generate_app_configs_file = os.environ.get('APP_CONFIGS_FILE', 'False') # generate the APP_CONFIGS_FILE, if not exist already + write_file_if_not_exists = local_debug == 'True' or generate_app_configs_file == 'True' + + success, dict = load_global_dict_from_file(app_configs, APP_CONFIGS_FILE, "APP_CONFIGS", write_file=write_file_if_not_exists) + + if success: + app_configs = dict # overwrite code-defaults (from local or external settings) + #else app_configs = + + return + +init_app_configs() # load from JSON file (local or remote) with code-defaults otherwise + +# lutzapps - add kohya_ss support and handle the required local "venv" within the "kohya_ss" app folder def ensure_kohya_local_venv_is_symlinked() -> tuple[bool, str]: + ### create a folder symlink for kohya's "local" 'venv' dir # as kohya_ss' "setup.sh" assumes a "local" VENV under "/workspace/kohya_ss/venv", # we will create a folder symlink "/workspace/kohya_ss/venv" -> "/workspace/bkohya" - # to our global VENV and rename the original "venv" folder to "venv(BAK)" + # to our global VENV and rename the original "venv" folder to "venv(BAK)", if any exists, + # will we not the case normally. - if not DEBUG_SETTINGS['use_bkohya_local_venv_symlink'] == "1": - return True, "" # not fix the local KOHYA_SS VENV + if not DEBUG_SETTINGS['create_bkohya_to_local_venv_symlink']: + return True, "" # not fix the local KOHYA_SS VENV requirement import shutil @@ -251,13 +467,19 @@ def ensure_kohya_local_venv_is_symlinked() -> tuple[bool, str]: bapp_app_path = app_configs[bapp_name]["app_path"] # '/workspace/kohya_ss' bapp_app_path_venv = f"{bapp_app_path}/venv" # '/workspace/kohya_ss/venv' - if not os.path.exists(bapp_app_path): # kohya is not installed - return True, "" # no need to fix the local KOHYA VENV - - # kohya installed and has a local "venv" folder - if os.path.exists(bapp_app_path_venv) and os.path.isdir(bapp_app_path_venv): + name = app_configs[bapp_name]["name"] - # check if this local VENV is a folderlink to target our bkohya global VENV to venv_path + if not os.path.exists(bapp_app_path): # kohya is not installed + return True, f"{name} is not installed." # no need to fix the local KOHYA VENV + + # check the src-folder of 'bkohya' downloaded VENV exists + if not os.path.exists(bapp_venv_path): # src_path to bkohya downloaded venv does NOT exists + return True, f"{name} VENV is not installed." # no need to fix the local KOHYA VENV, as the global KOHYA VENV does not exist + + # kohya_ss is installed + if os.path.isdir(bapp_app_path_venv): # and has a local "venv" folder + + # check if this local VENV is a folderlink to target the bkohya global VENV to venv_path if os.path.islink(bapp_app_path_venv): success_message = f"kohya_ss local venv folder '{bapp_app_path_venv}' is already symlinked" @@ -273,18 +495,17 @@ def ensure_kohya_local_venv_is_symlinked() -> tuple[bool, str]: i += 1 suffix = str(i) - bak_venv_path += suffix # free target bame for "rename" + bak_venv_path += suffix # free target name for "rename"(move) operation of the folder shutil.move(bapp_app_path_venv, bak_venv_path) # move=rename print(f"local venv folder '{bapp_app_path_venv}' detected and renamed to '{bak_venv_path}'") + # now the path to the local "venv" is free, if it was already created it is now renamed ### create a folder symlink for kohya's "local" venv dir - # check the src-folder to kohya downloaded venv exists - if os.path.exists(bapp_venv_path): # src_path to bkohya downloaded venv exists - # create a folder symlink for kohya local venv dir - os.symlink(bapp_venv_path, bapp_app_path_venv, target_is_directory=True) - success_message = f"created a symlink for kohya_ss local venv folder: '{bapp_venv_path}' -> '{bapp_app_path_venv}'" - print(success_message) + # create a folder symlink for kohya local venv dir + os.symlink(bapp_venv_path, bapp_app_path_venv, target_is_directory=True) + success_message = f"created a symlink for kohya_ss local venv folder: '{bapp_app_path_venv}' -> '{bapp_venv_path}'" + print(success_message) return True, success_message @@ -295,4 +516,42 @@ def ensure_kohya_local_venv_is_symlinked() -> tuple[bool, str]: return False, error_message # lutzapps - add kohya_ss venv support -ensure_kohya_local_venv_is_symlinked() \ No newline at end of file +ensure_kohya_local_venv_is_symlinked() + +# some verification steps of the VENV setup of the "kohya_ss" app: +# even if it "looks" like the "venv" is in a local sub-folder of the "kohya_ss" dir, +# this location is only "aliased/symlinked" there from the globally downloaded +# tarfile "bkohya.tar.gz" which was expanded spearately into the folder "/workspace/bkohya". +# So the VENV can be redownloaded separately from the github app at "/workspace/kohya_ss" + # root@9452ad7f4cd6:/workspace/kohya_ss# python --version + # Python 3.11.10 + # root@fe889cc68f5a:/workspace/kohya_ss# pip --version + # pip 24.3.1 from /usr/local/lib/python3.11/dist-packages/pip (python 3.11) + # + # root@9452ad7f4cd6:/workspace/kohya_ss# python3 --version + # Python 3.11.10 + # root@fe889cc68f5a:/workspace/kohya_ss# pip3 --version + # pip 24.3.1 from /usr/local/lib/python3.11/dist-packages/pip (python 3.11) + # + # root@9452ad7f4cd6:/workspace/kohya_ss# ls venv -la + # lrwxr-xr-x 1 root root 17 Nov 8 00:06 venv -> /workspace/bkohya + # + # root@9452ad7f4cd6:/workspace/kohya_ss# source venv/bin/activate + # + # (bkohya) root@9452ad7f4cd6:/workspace/kohya_ss# ls venv/bin/python* -la + # lrwxr-xr-x 1 root root 10 Nov 8 00:48 venv/bin/python -> python3.10 + # lrwxr-xr-x 1 root root 10 Nov 8 00:48 venv/bin/python3 -> python3.10 + # lrwxr-xr-x 1 root root 19 Nov 8 00:48 venv/bin/python3.10 -> /usr/bin/python3.10 + # + # (bkohya) root@9452ad7f4cd6:/workspace/kohya_ss# python --version + # Python 3.10.12 + # (bkohya) root@fe889cc68f5a:/workspace/kohya_ss# pip --version + # pip 22.0.2 from /workspace/venv/lib/python3.10/site-packages/pip (python 3.10) + # + # (bkohya) root@9452ad7f4cd6:/workspace/kohya_ss# python3 --version + # Python 3.10.12 + # (bkohya) root@fe889cc68f5a:/workspace/kohya_ss# pip3 --version + # pip 22.0.2 from /workspace/venv/lib/python3.10/site-packages/pip (python 3.10) + # + # (bkohya) root@9452ad7f4cd6:/workspace/kohya_ss# deactivate + # root@9452ad7f4cd6:/workspace/kohya_ss# diff --git a/official-templates/better-ai-launcher/app/utils/app_utils.py b/official-templates/better-ai-launcher/app/utils/app_utils.py index b484bc4..3001754 100644 --- a/official-templates/better-ai-launcher/app/utils/app_utils.py +++ b/official-templates/better-ai-launcher/app/utils/app_utils.py @@ -13,6 +13,8 @@ import xml.etree.ElementTree as ET import time import datetime import shutil +from utils.app_configs import (DEBUG_SETTINGS, pretty_dict, init_app_configs, init_debug_settings, write_debug_setting, ensure_kohya_local_venv_is_symlinked) +from utils.model_utils import (get_sha256_hash_from_file) INSTALL_STATUS_FILE = '/tmp/install_status.json' @@ -180,41 +182,45 @@ import time # yield (out_line.rstrip(), err_line.rstrip()) -# this ist the v2 ("fast") version for "download_and_unpack_venv()" - can be (de-)/activated in DEBUG_SETTINGS dict -def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_message) -> tuple[bool, str]: +# this is the v2 ("fast") version for "download_and_unpack_venv()" - can be (de-)/activated in DEBUG_SETTINGS dict +def download_and_unpack_venv_v2(app_name:str, app_configs:dict, send_websocket_message) -> tuple[bool, str]: + # load the latest configured DEBUG_SETTINGS from the stored setting of the DEBUG_SETTINGS_FILE + init_debug_settings() # reload latest DEBUG_SETTINGS + # as this could overwrite the APP_CONFIGS_MANIFEST_URL, we reload the app_configs global dict + # from whatever Url is now defined + init_app_configs() # reload lastest app_configs dict + app_config = app_configs.get(app_name) if not app_config: return False, f"App '{app_name}' not found in configurations." venv_path = app_config['venv_path'] - app_path = app_config['app_path'] download_url = app_config['download_url'] - total_size = app_config['size'] + archive_size = app_config['archive_size'] + tar_filename = os.path.basename(download_url) workspace_dir = '/workspace' downloaded_file = os.path.join(workspace_dir, tar_filename) - from utils.app_configs import (DEBUG_SETTINGS, pretty_dict, init_debug_settings, write_debug_setting, ensure_kohya_local_venv_is_symlinked) - # load the latest configured DEBUG_SETTINGS from the stored setting of the DEBUG_SETTINGS_FILE - init_debug_settings() - # show currently using DEBUG_SETTINGS - print(f"\nCurrently using 'DEBUG_SETTINGS':\n{pretty_dict(DEBUG_SETTINGS)}") - write_debug_setting('tar_filename', tar_filename) write_debug_setting('download_url', download_url) try: + if DEBUG_SETTINGS['skip_to_github_stage']: + success, message = clone_application(app_config,send_websocket_message) + return success, message + save_install_status(app_name, 'in_progress', 0, 'Downloading') - send_websocket_message('install_log', {'app_name': app_name, 'log': f'Downloading {total_size / (1024 * 1024):.2f} MB ...'}) + send_websocket_message('install_log', {'app_name': app_name, 'log': f'Downloading {archive_size / (1024 * 1024):.2f} MB ...'}) start_time_download = time.time() # debug with existing local cached TAR file if os.path.exists(downloaded_file): - write_debug_setting('used_local_tar', "1") # indicate using cached TAR file - send_websocket_message('install_log', {'app_name': app_name, 'log': f"Used cached local tarfile '{downloaded_file}'"}) + write_debug_setting('used_local_tarfile', True) # indicate using cached TAR file + send_websocket_message('used_local_tarfile', {'app_name': app_name, 'log': f"Used cached local tarfile '{downloaded_file}'"}) else: - write_debug_setting('used_local_tar', "0") # indicate no cached TAR file found + write_debug_setting('used_local_tarfile', False) # indicate no cached TAR file found try: ### download with ARIA2C @@ -275,8 +281,8 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m gid = match.group(1) # e.g., "cd57da" downloaded_size_value = match.group(2) # e.g., "2.1" downloaded_size_unit = match.group(3) # e.g., "GiB" - total_size_value = match.group(4) # e.g., "4.0" - total_size_unit = match.group(5) # e.g., "GiB" + total_size_value = match.group(4) # e.g., "4.0" (this could replace the 'archive_size' from the manifest) + total_size_unit = match.group(5) # e.g., "GiB" (with calculation to bytes, but not sure if its rounded) percentage = int(match.group(6)) # e.g., "53" connection_count = int(match.group(7)) # e.g., "16" download_rate_value = match.group(8) # e.g., "1.9" @@ -296,8 +302,8 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m ### original code #speed = downloaded_size / elapsed_time # bytes/sec - #percentage = (downloaded_size / total_size) * 100 - #eta = (total_size - downloaded_size) / speed if speed > 0 else 0 # sec + #percentage = (downloaded_size / archive_size) * 100 + #eta = (archive_size - downloaded_size) / speed if speed > 0 else 0 # sec send_websocket_message('install_progress', { 'app_name': app_name, @@ -329,7 +335,7 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m os.remove(f"{tar_filename}.aria2") except Exception as e: - error_msg = f"ERROR in download_and_unpack_venv_fastversion():download with ARIA2C\ncmdline: '{cmd_line}'\nException: {str(e)}" + error_msg = f"ERROR in download_and_unpack_venv_v2():download with ARIA2C\ncmdline: '{cmd_line}'\nException: {str(e)}" print(error_msg) error_message = f"Downloading VENV failed: {download_process.stderr.read() if download_process.stderr else 'Unknown error'}" @@ -356,8 +362,8 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m # if elapsed_time > 0: # speed = downloaded_size / elapsed_time - # percentage = (downloaded_size / total_size) * 100 - # eta = (total_size - downloaded_size) / speed if speed > 0 else 0 + # percentage = (downloaded_size / archive_size) * 100 + # eta = (archive_size - downloaded_size) / speed if speed > 0 else 0 # send_websocket_message('install_progress', { # 'app_name': app_name, @@ -375,29 +381,72 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m return False, error_message - send_websocket_message('install_log', {'app_name': app_name, 'log': 'Download completed. Starting unpacking...'}) - send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 100, 'stage': 'Download Complete'}) + send_websocket_message('install_log', {'app_name': app_name, 'log': 'Download completed. Starting Verification ...'}) + # we use a 99% progress and indicate 1% for Verification against the files SHA256 hash + send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 99, 'stage': 'Downloading'}) total_duration_download = f"{datetime.timedelta(seconds=int(time.time() - start_time_download))}" write_debug_setting('total_duration_download', total_duration_download) print(f"download did run {total_duration_download} for app '{app_name}'") + ### VERIFY stage + # + # Create TAR from the VENV current directory: + # IMPORTANT: cd INTO the folder you want to compress, as we use "." for source folder, + # to avoid having the foldername in the TAR file !!! + # PV piping is "nice-to-have" and is only used for showing "Progress Values" during compressing + # + # cd /workspace/bkohya + # #tar -czf | pv > /workspace/bkohya.tar.gz . (not the smallest TAR)# + # tar -cvf - . | gzip -9 - | pv > /workspace/bkohya.tar.gz + # + # afterwards create the SHA256 hash from this TAR with + # shasum -a 256 bkohya.tar.gz + # + # also report the uncompressed size from the current VENV directory, + # we need that as the 100% base for the progress indicators when uncompressing the TAR + + + # verify the downloaded TAR file against its SHA256 hash value from the manifest + + download_sha256_hash = app_config["sha256_hash"].lower() # get the sha256_hash from the app manifest + file_verified = False + + print(f"getting SHA256 Hash for '{downloaded_file}'") + successfull_HashGeneration, file_sha256_hash = get_sha256_hash_from_file(downloaded_file) + + if successfull_HashGeneration and file_sha256_hash.lower() == download_sha256_hash.lower(): + file_verified = True + message = f"Downloaded file '{os.path.basename(downloaded_file)}' was successfully (SHA256) verified." + print(message) + + else: + if successfull_HashGeneration: # the generated SHA256 file hash did not match against the metadata hash + error_message = f"The downloaded file '{os.path.basename(downloaded_file)}' has DIFFERENT \nSHA256: {file_sha256_hash} as in the manifest\nFile is possibly corrupted and was DELETED!" + print(error_message) + + os.remove(downloaded_file) # delete corrupted, downloaded file + + + else: # NOT successful, the hash contains the Exception + error_msg = file_sha256_hash + error_message = f"Exception occured while generating the SHA256 hash for '{downloaded_file}':\n{error_msg}" + print(error_message) + + if not file_verified: + send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message}) + save_install_status(app_name, 'failed', 0, 'Failed') + + return False, error_message + + send_websocket_message('install_log', {'app_name': app_name, 'log': 'Verification completed. Starting unpacking ...'}) + send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 100, 'stage': 'Download Complete'}) + + ### Decompression Stage (Unpacking the downloaded VENV) start_time_unpack = time.time() - # lutzapps - fix TAR packaging bug (compressed from the workspace root instead of bkohya VENV folder) - # e.g. "bkohya/bin/activate", together with venv_path ("/workspace/bkohya") ends up as "/workspace/bkohya/bkohya/bin/activate" - # TODO: need to repackage Kohya VENV correctly and then remove this fix!!! - - if app_name == "bkohya" and DEBUG_SETTINGS['use_bkohya_tar_folder_fix'] == "1": - venv_path = "/workspace" # extracts then correctly to '/workspace/bkohya', instead of '/workspace/bkohya/bkohya' - - # Create TAR from the VENV current directory: - # cd ~/Projects/Docker/madiator/workspace/bkohya - # [tar -czf | pv > ~/Projects/Docker/madiator/workspace/bkohya.tar.gz . (not the smallest TAR)] - # tar -cvf - . | gzip -9 - | pv > ~/Projects/Docker/madiator/workspace/bkohya.tar.gz - # Ensure the venv directory exists os.makedirs(f"{venv_path}/", exist_ok=True) # append trailing "/" to make sure the last sub-folder is created @@ -419,11 +468,8 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m # 'bforge': 7689838771 # 'bkohya': 12192767148 - uncompressed_size_bytes = DEBUG_SETTINGS["manifests"][app_name]["venv_uncompressed_size"] + uncompressed_size_bytes = app_config["venv_uncompressed_size"] - #sha256_hash = DEBUG_SETTINGS["manifests"][app_name]["sha256_hash"] - # TODO: create with 'shasum -a 256 xxx.tar.gz' - ### NOTE: as it turns out GZIP has problems with files bigger than 2 or 4 GB due to internal field bit restrictions # cmd_line = f"gzip -l {downloaded_file}" # e.g. for 'ba1111.tar.gz' @@ -601,7 +647,7 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m # else any other line in stdout (which we not process) except Exception as e: - error_msg = f"ERROR in download_and_unpack_venv_fastversion():\ncmdline: '{cmd_line}'\nException: {str(e)}" + error_msg = f"ERROR in download_and_unpack_venv_v2():\ncmdline: '{cmd_line}'\nException: {str(e)}" print(error_msg) decompression_process.wait() # let the process finish @@ -621,8 +667,11 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m send_websocket_message('install_progress', {'app_name': app_name, 'percentage': 100, 'stage': 'Unpacking Complete'}) - print(f"'DEBUG_SETTINGS' after this run:\n{pretty_dict(DEBUG_SETTINGS)}") + ### installing the App from GITHUB + # Clone the repository if it doesn't exist + success, message = clone_application(app_name) + print(f"'DEBUG_SETTINGS' after this run:\n{pretty_dict(DEBUG_SETTINGS)}") ### original "v1" code (very slow code because of STATISTICS glory @@ -647,66 +696,29 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m # process.wait() # rc = process.returncode - ### installing the App from GITHUB # Clone the repository if it doesn't exist - if not os.path.exists(app_path): - send_websocket_message('install_log', {'app_name': app_name, 'log': 'Cloning repository...'}) - - repo_url = '' - if app_name == 'bcomfy': - repo_url = 'https://github.com/comfyanonymous/ComfyUI.git' - elif app_name == 'bforge': - repo_url = 'https://github.com/lllyasviel/stable-diffusion-webui-forge.git' - elif app_name == 'ba1111': - repo_url = 'https://github.com/AUTOMATIC1111/stable-diffusion-webui.git' - elif app_name == 'bkohya': # lutzapps - added new Kohya app - repo_url = 'https://github.com/bmaltais/kohya_ss.git' - - try: # add a repo assignment for Kohya - repo = git.Repo.clone_from(repo_url, app_path, progress=lambda op_code, cur_count, max_count, message: send_websocket_message('install_log', { - 'app_name': app_name, - 'log': f"Cloning: {cur_count}/{max_count} {message}" - })) - send_websocket_message('install_log', {'app_name': app_name, 'log': 'Repository cloned successfully.'}) - - # lutzapps - make sure we use Kohya with FLUX support - if app_name == 'bkohya': - branch_name = "sd3-flux.1" # this branch also uses a "sd-scripts" branch "SD3" automatically - repo.git.checkout(branch_name) - - # Clone ComfyUI-Manager for Better ComfyUI - if app_name == 'bcomfy': - custom_nodes_path = os.path.join(app_path, 'custom_nodes') - os.makedirs(custom_nodes_path, exist_ok=True) - comfyui_manager_path = os.path.join(custom_nodes_path, 'ComfyUI-Manager') - if not os.path.exists(comfyui_manager_path): - send_websocket_message('install_log', {'app_name': app_name, 'log': 'Cloning ComfyUI-Manager...'}) - git.Repo.clone_from('https://github.com/ltdrdata/ComfyUI-Manager.git', comfyui_manager_path) - send_websocket_message('install_log', {'app_name': app_name, 'log': 'ComfyUI-Manager cloned successfully.'}) - - except git.exc.GitCommandError as e: - send_websocket_message('install_log', {'app_name': app_name, 'log': f'Error cloning repository: {str(e)}'}) - return False, f"Error cloning repository: {str(e)}" - - if app_name == 'bkohya': # create a folder link for kohya_ss local "venv" - ensure_kohya_local_venv_is_symlinked() + success, error_message = clone_application(app_name, send_websocket_message) # Clean up the downloaded file send_websocket_message('install_log', {'app_name': app_name, 'log': 'Cleaning up...'}) # lutzapps - debug with local TAR # do NOT delete the Kohya venv - if DEBUG_SETTINGS["delete_tarfile_after_download"] == "1": # this is the default, but can be overwritten + if DEBUG_SETTINGS["delete_tar_file_after_download"]: # this is the default, but can be overwritten os.remove(downloaded_file) send_websocket_message('install_log', {'app_name': app_name, 'log': 'Installation complete. Refresh page to start app'}) - save_install_status(app_name, 'completed', 100, 'Completed') - send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."}) - return True, "Virtual environment installed successfully." + if success: + save_install_status(app_name, 'completed', 100, 'Completed') + send_websocket_message('install_complete', {'app_name': app_name, 'status': 'success', 'message': "Virtual environment installed successfully."}) + return True, "Virtual environment installed successfully." + else: + return False, error_message + except requests.RequestException as e: - error_message = f"Download failed: {str(e)}" + error_message = f"Download/Decompression failed: {str(e)}" send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message}) save_install_status(app_name, 'failed', 0, 'Failed') return False, error_message @@ -716,8 +728,143 @@ def download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_m send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message}) return False, error_message +### installing the App from GITHUB +# Clone the repository if it doesn't exist +def clone_application(app_config:dict, send_websocket_message) -> tuple[bool , str]: + try: + app_name = app_config['id'] + app_path = app_config['app_path'] -def download_and_unpack_venv(app_name, app_configs, send_websocket_message): + if not os.path.exists(app_path): # only install new apps + repo_url = app_config['repo_url'] + branch_name = app_config['branch_name'] + if branch_name == "": # use the default branch + branch_name = "master" + clone_recursive = app_config['clone_recursive'] + + send_websocket_message('install_log', {'app_name': app_name, 'log': f"Cloning repository '{repo_url}' branch '{branch_name}' recursive={clone_recursive} ..."}) + + repo = git.Repo.clone_from(repo_url, app_path, # first 2 params are fix, then use named params + #branch=branch_name, # if we provide a branch here, we ONLY get this branch downloaded + # we want ALL branches, so we can easy checkout different versions from kohya_ss late, without re-downloading + recursive=clone_recursive, # include cloning submodules recursively (if needed as with Kohya) + progress=lambda op_code, cur_count, max_count, message: send_websocket_message('install_log', { + 'app_name': app_name, + 'log': f"Cloning: {cur_count}/{max_count} {message}" + })) + + + send_websocket_message('install_log', {'app_name': app_name, 'log': 'Repository cloned successfully.'}) + + # lutzapps - make sure we use Kohya with FLUX support + if not branch_name == "master": + repo.git.checkout(branch_name) # checkout the "sd3-flux.1" branch, but could later switch back to "master" easy + # the setup can be easy verified with git, here e.g. for the "kohya_ss" app: + # root@fe889cc68f5a:~# cd /workspace/kohya_ss + # root@fe889cc68f5a:/workspace/kohya_ss# git branch + # master + # * sd3-flux.1 + # root@fe889cc68f5a:/workspace/kohya_ss# cd sd-scripts + # root@fe889cc68f5a:/workspace/kohya_ss/sd-scripts# git branch + # * (HEAD detached at b8896aa) + # main + # + # in the case of kohya_ss we need to fix a bug in the 'setup.sh' file, + # where they forgot to adapt the branch name from "master" to "sd3-flux.1" + # in the "#variables" section for refreshing kohya via git with 'setup.sh' + if app_name == 'bkohya': + success, message = update_kohya_setup_sh(app_path) # patch the 'setup.sh' file + print(message) # shows, if the patch was needed, and apllied successfully + else: # refresh app + if app_path['refresh']: # app wants auto-refreshes + # TODO: implement app refreshes via git pull or, in the case of 'kohya_ss' via "setup.sh" + message = f"Refreshing of app '{app_name}' is NYI" + print(message) + + # Clone ComfyUI-Manager and other defined custom_nodes for Better ComfyUI + if app_name == 'bcomfy': + # install all defined custom nodes + custom_nodes_path = os.path.join(app_path, 'custom_nodes') + os.makedirs(f"{custom_nodes_path}/", exist_ok=True) # append a trailing slash to be sure last dir is created + for custom_node in app_config['custom_nodes']: + name = custom_node['name'] + path = custom_node['path'] + repo_url = custom_node['repo_url'] + custom_node_path = os.path.join(custom_nodes_path, path) + + if not os.path.exists(custom_node_path): # only install new custom nodes + send_websocket_message('install_log', {'app_name': app_name, 'log': f"Cloning '{name}' ..."}) + git.Repo.clone_from(repo_url, custom_node_path) + send_websocket_message('install_log', {'app_name': app_name, 'log': f"'{name}' cloned successfully."}) + + # install requirements + venv_path = app_config['venv_path'] + #app_path = app_config['app_path'] # already defined + + try: + # Activate the virtual environment and run the commands + activate_venv = f"source {venv_path}/bin/activate" + change_dir_command = f"cd {custom_node_path}" + pip_install_command = "pip install -r requirements.txt" + + full_command = f"{activate_venv} && {change_dir_command} && {pip_install_command}" + + # TODO: rewrite this without shell + process = subprocess.Popen(full_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, executable='/bin/bash') + output, _ = process.communicate() + + if process.returncode == 0: + return True, f"Custom node requirements were successfully installed. Output: {output.decode('utf-8')}" + else: + return False, f"Error in custom node requirements installation. Output: {output.decode('utf-8')}" + except Exception as e: + return False, f"Error installing custom node requirements: {str(e)}" + + + except git.exc.GitCommandError as e: + send_websocket_message('install_log', {'app_name': app_name, 'log': f'Error cloning repository: {str(e)}'}) + return False, f"Error cloning repository: {str(e)}" + except Exception as e: + send_websocket_message('install_log', {'app_name': app_name, 'log': f'Error cloning repository: {str(e)}'}) + return False, f"Error cloning repository: {str(e)}" + + + if app_name == 'bkohya': # create a folder link for kohya_ss local "venv" + success, message = ensure_kohya_local_venv_is_symlinked() + if not success: # symlink not created, but still success=True and only a warning, can be fixed manually + message = f"{app_config['name']} was cloned and patched successfully, but the symlink to the local venv returned following problem:\n{message}" + else: + message = f"'{app_name}' was cloned successfully." + + return True, message + +def update_kohya_setup_sh(app_path:str) -> tuple[bool, str]: + try: + # patch 'setup.sh' within the kohya_ss main folder for BRANCH="sd3-flux.1" + setup_sh_path = os.path.join(app_path, 'setup.sh') + if not os.path.exists(setup_sh_path): + return False, f"file '{setup_sh_path}' was not found" + + with open(setup_sh_path, 'r') as file: + content = file.read() + + # Use regex to search & replace wrong branch variable in the file + patched_content = re.sub(r'BRANCH="master"', 'BRANCH="sd3-flux.1"', content) + + if patched_content == content: + message = f"'{setup_sh_path}' already fine, patch not needed." + else: + with open(setup_sh_path, 'w') as file: + file.write(patched_content) + + message = f"'{setup_sh_path}' needed patch, successfully patched." + + return True, message + + except Exception as e: + return False, str(e) + +def download_and_unpack_venv_v1(app_name, app_configs, send_websocket_message): app_config = app_configs.get(app_name) if not app_config: return False, f"App '{app_name}' not found in configurations." @@ -725,14 +872,14 @@ def download_and_unpack_venv(app_name, app_configs, send_websocket_message): venv_path = app_config['venv_path'] app_path = app_config['app_path'] download_url = app_config['download_url'] - total_size = app_config['size'] + archive_size = app_config['size'] tar_filename = os.path.basename(download_url) workspace_dir = '/workspace' downloaded_file = os.path.join(workspace_dir, tar_filename) try: save_install_status(app_name, 'in_progress', 0, 'Downloading') - send_websocket_message('install_log', {'app_name': app_name, 'log': f'Starting download of {total_size / (1024 * 1024):.2f} MB...'}) + send_websocket_message('install_log', {'app_name': app_name, 'log': f'Starting download of {archive_size / (1024 * 1024):.2f} MB...'}) # lutzapps - debug with existing local TAR if not os.path.exists(downloaded_file): @@ -753,8 +900,8 @@ def download_and_unpack_venv(app_name, app_configs, send_websocket_message): if elapsed_time > 0: speed = downloaded_size / elapsed_time - percentage = (downloaded_size / total_size) * 100 - eta = (total_size - downloaded_size) / speed if speed > 0 else 0 + percentage = (downloaded_size / archive_size) * 100 + eta = (archive_size - downloaded_size) / speed if speed > 0 else 0 send_websocket_message('install_progress', { 'app_name': app_name, @@ -869,31 +1016,34 @@ def download_and_unpack_venv(app_name, app_configs, send_websocket_message): send_websocket_message('install_complete', {'app_name': app_name, 'status': 'error', 'message': error_message}) return False, error_message -### this is the function wgich switches between v0 and v1 debug setting for comparison -def download_and_unpack_venv(app_name, app_configs, send_websocket_message) -> tuple[bool, str]: - from app_configs import DEBUG_SETTINGS, write_debug_setting +### this is the function which switches between v0 and v1 debug setting for comparison +def download_and_unpack_venv(app_name:str, app_configs:dict, send_websocket_message) -> tuple[bool, str]: + from utils.app_configs import DEBUG_SETTINGS, write_debug_setting installer_codeversion = DEBUG_SETTINGS['installer_codeversion'] # read from DEBUG_SETTINGS - print(f"download_and_unpack_venv v{installer_codeversion} STARTING for '{app_name}'") + print(f"download_and_unpack_venv_{installer_codeversion} STARTING for '{app_name}'") import time start_time = time.time() - if installer_codeversion == "1": - download_and_unpack_venv(app_name, app_configs, send_websocket_message) - elif installer_codeversion == "2": - download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_message) + if installer_codeversion == "v1": + success, message = download_and_unpack_venv_v1(app_name, app_configs, send_websocket_message) + elif installer_codeversion == "v2": + success, message = download_and_unpack_venv_v2(app_name, app_configs, send_websocket_message) else: - print(f"unknown 'installer_codeversion' v{installer_codeversion} found, nothing run for app '{app_name}'") + error_msg = f"unknown 'installer_codeversion' {installer_codeversion} found, nothing run for app '{app_name}'" + print(error_msg) + success = False + message = error_msg total_duration = f"{datetime.timedelta(seconds=int(time.time() - start_time))}" write_debug_setting('app_name', app_name) write_debug_setting('total_duration', total_duration) - print(f"download_and_unpack_venv v{installer_codeversion} did run {total_duration} for app '{app_name}'") - + print(f"download_and_unpack_venv_v{installer_codeversion} did run {total_duration} for app '{app_name}'") + return success, message def fix_custom_nodes(app_name, app_configs): if app_name != 'bcomfy': @@ -921,54 +1071,9 @@ def fix_custom_nodes(app_name, app_configs): return False, f"Error fixing custom nodes: {str(e)}" # Replace the existing install_app function with this updated version -def install_app(app_name, app_configs, send_websocket_message): +def install_app(app_name:str, app_configs:dict, send_websocket_message) -> tuple[bool, str]: if app_name in app_configs: - #return download_and_unpack_venv(app_name, app_configs, send_websocket_message) - return download_and_unpack_venv_fastversion(app_name, app_configs, send_websocket_message) + success, message = download_and_unpack_venv(app_name, app_configs, send_websocket_message) + return success, message else: return False, f"Unknown app: {app_name}" - -# unused function -def onsolate_update_model_symlinks(): - # lutzapps - CHANGE #7 - use the new "shared_models" module for app model sharing - # remove this whole now unused function - return "replaced by utils.shared_models.update_model_symlinks()" - - shared_models_dir = '/workspace/shared_models' - apps = { - 'stable-diffusion-webui': '/workspace/stable-diffusion-webui/models', - 'stable-diffusion-webui-forge': '/workspace/stable-diffusion-webui-forge/models', - 'ComfyUI': '/workspace/ComfyUI/models' - } - model_types = ['Stable-diffusion', 'VAE', 'Lora', 'ESRGAN'] - - for model_type in model_types: - shared_model_path = os.path.join(shared_models_dir, model_type) - - if not os.path.exists(shared_model_path): - continue - - for app, app_models_dir in apps.items(): - if app == 'ComfyUI': - if model_type == 'Stable-diffusion': - app_model_path = os.path.join(app_models_dir, 'checkpoints') - elif model_type == 'Lora': - app_model_path = os.path.join(app_models_dir, 'loras') - elif model_type == 'ESRGAN': - app_model_path = os.path.join(app_models_dir, 'upscale_models') - else: - app_model_path = os.path.join(app_models_dir, model_type.lower()) - else: - app_model_path = os.path.join(app_models_dir, model_type) - - # Create the app model directory if it doesn't exist - os.makedirs(app_model_path, exist_ok=True) - - # Create symlinks for each file in the shared model directory - for filename in os.listdir(shared_model_path): - src = os.path.join(shared_model_path, filename) - dst = os.path.join(app_model_path, filename) - if os.path.isfile(src) and not os.path.exists(dst): - os.symlink(src, dst) - - print("Model symlinks updated.") diff --git a/official-templates/better-ai-launcher/app/utils/model_utils.py b/official-templates/better-ai-launcher/app/utils/model_utils.py index 69a35b1..d6ff370 100644 --- a/official-templates/better-ai-launcher/app/utils/model_utils.py +++ b/official-templates/better-ai-launcher/app/utils/model_utils.py @@ -71,7 +71,7 @@ def check_huggingface_url(url): return True, repo_id, filename, folder_name, branch_name -def download_model(url, model_name, model_type, civitai_token=None, hf_token=None, version_id=None, file_index=None): +def download_model(url, model_name, model_type, civitai_token=None, hf_token=None, version_id=None, file_index=None) -> tuple[bool, str]: ensure_shared_folder_exists() is_civitai, is_civitai_api, model_id, _ = check_civitai_url(url) is_huggingface, repo_id, hf_filename, hf_folder_name, hf_branch_name = check_huggingface_url(url) # TODO: double call @@ -95,7 +95,7 @@ def download_model(url, model_name, model_type, civitai_token=None, hf_token=Non return success, message # lutzapps - added SHA256 checks for already existing ident and downloaded HuggingFace model -def download_civitai_model(url, model_name, model_type, civitai_token, version_id=None, file_index=None): +def download_civitai_model(url, model_name, model_type, civitai_token, version_id=None, file_index=None) -> tuple[bool, str]: try: is_civitai, is_civitai_api, model_id, url_version_id = check_civitai_url(url) @@ -186,7 +186,7 @@ def get_sha256_hash_from_file(file_path:str) -> tuple[bool, str]: for byte_block in iter(lambda: f.read(4096), b""): sha256_hash.update(byte_block) - return True, sha256_hash.hexdigest().upper() + return True, sha256_hash.hexdigest().lower() except Exception as e: return False, str(e) @@ -247,7 +247,7 @@ def get_modelfile_hash_and_ident_existing_modelfile_exists(model_name:str, model raise NotImplementedError("Copying a non-LFS file is not implemented.") lfs = repo_file.lfs # BlobLfsInfo class instance - download_sha256_hash = lfs.sha256.upper() + download_sha256_hash = lfs.sha256.lower() print(f"Metadata from RepoFile LFS '{repo_file.rfilename}'") print(f"SHA256: {download_sha256_hash}") @@ -283,8 +283,8 @@ def get_modelfile_hash_and_ident_existing_modelfile_exists(model_name:str, model # if NOT successful, the hash contains the Exception print(f"SHA256 hash generated from local file: '{model_path}'\n{model_sha256_hash}") - if successfull_HashGeneration and model_sha256_hash == download_sha256_hash: - message = f"Existing and ident model aleady found for '{os.path.basename(model_path)}'" + if successfull_HashGeneration and model_sha256_hash.lower() == download_sha256_hash.lower(): + message = f"Existing and ident model already found for '{os.path.basename(model_path)}'" print(message) send_websocket_message('model_download_progress', { @@ -315,7 +315,7 @@ def get_modelfile_hash_and_ident_existing_modelfile_exists(model_name:str, model # lutzapps - added SHA256 checks for already existing ident and downloaded HuggingFace model -def download_huggingface_model(url, model_name, model_type, repo_id, hf_filename, hf_folder_name, hf_branch_name, hf_token=None): +def download_huggingface_model(url, model_name, model_type, repo_id, hf_filename, hf_folder_name, hf_branch_name, hf_token=None) -> tuple[bool, str]: try: from huggingface_hub import hf_hub_download @@ -372,7 +372,7 @@ def download_huggingface_model(url, model_name, model_type, repo_id, hf_filename # lutzapps - added SHA256 check for downloaded CivitAI model -def download_file(url, download_sha256_hash, file_path, headers=None): +def download_file(url, download_sha256_hash, file_path, headers=None) -> tuple[bool, str]: try: response = requests.get(url, stream=True, headers=headers) response.raise_for_status() @@ -428,7 +428,7 @@ def check_downloaded_modelfile(model_path:str, download_sha256_hash:str, platfor }) successfull_HashGeneration, model_sha256_hash = get_sha256_hash_from_file(model_path) - if successfull_HashGeneration and model_sha256_hash == download_sha256_hash: + if successfull_HashGeneration and model_sha256_hash.lower() == download_sha256_hash.lower(): send_websocket_message('model_download_progress', { 'percentage': 100, 'stage': 'Complete', diff --git a/official-templates/better-ai-launcher/app/utils/shared_models.py b/official-templates/better-ai-launcher/app/utils/shared_models.py index c014c02..f29ec04 100644 --- a/official-templates/better-ai-launcher/app/utils/shared_models.py +++ b/official-templates/better-ai-launcher/app/utils/shared_models.py @@ -6,7 +6,7 @@ import time from flask import jsonify from utils.websocket_utils import (send_websocket_message, active_websockets) -from utils.app_configs import (get_app_configs, init_global_dict_from_file, pretty_dict) +from utils.app_configs import (get_app_configs, load_global_dict_from_file, pretty_dict) ### shared_models-v0.9.2 by lutzapps, Nov 5th 2024 ### @@ -189,7 +189,9 @@ SHARED_MODEL_FOLDERS = { # helper function called by "inline"-main() and ensure_shared_models_folders() def init_shared_models_folders(send_SocketMessage:bool=True): global SHARED_MODEL_FOLDERS - init_global_dict_from_file(SHARED_MODEL_FOLDERS, SHARED_MODEL_FOLDERS_FILE, "SHARED_MODEL_FOLDERS", SHARED_MODELS_DIR) + success, dict = load_global_dict_from_file(SHARED_MODEL_FOLDERS, SHARED_MODEL_FOLDERS_FILE, "SHARED_MODEL_FOLDERS", SHARED_MODELS_DIR) + if success: + SHARED_MODEL_FOLDERS = dict if os.path.exists(SHARED_MODEL_FOLDERS_FILE) and send_SocketMessage: send_websocket_message('extend_ui_helper', { @@ -341,7 +343,9 @@ def sync_with_app_configs_install_dirs(): # NOTE: this APP_INSTALL_DIRS_FILE is temporary synced with the app_configs dict def init_app_install_dirs(): global APP_INSTALL_DIRS - init_global_dict_from_file(APP_INSTALL_DIRS, APP_INSTALL_DIRS_FILE, "APP_INSTALL_DIRS", SHARED_MODELS_DIR) + success, dict = load_global_dict_from_file(APP_INSTALL_DIRS, APP_INSTALL_DIRS_FILE, "APP_INSTALL_DIRS", SHARED_MODELS_DIR) + if success: + APP_INSTALL_DIRS = dict return @@ -496,7 +500,9 @@ SHARED_MODEL_APP_MAP = { # which does a default mapping from app code or (if exists) from external JSON 'SHARED_MODEL_APP_MAP_FILE' file def init_shared_model_app_map(): global SHARED_MODEL_APP_MAP - init_global_dict_from_file(SHARED_MODEL_APP_MAP, SHARED_MODEL_APP_MAP_FILE, "SHARED_MODEL_APP_MAP", SHARED_MODELS_DIR) + success, dict = load_global_dict_from_file(SHARED_MODEL_APP_MAP, SHARED_MODEL_APP_MAP_FILE, "SHARED_MODEL_APP_MAP", SHARED_MODELS_DIR) + if success: + SHARED_MODEL_APP_MAP = dict return