From 12b8fa4c772aea2db08ebcc12a3b1d8dc70ef876 Mon Sep 17 00:00:00 2001
From: Fanghua-Yu <1901213025@pku.edu.cn>
Date: Thu, 25 Jan 2024 22:42:59 +0800
Subject: [PATCH] 20240125
---
.idea/.gitignore | 8 +
.idea/.name | 1 +
.idea/SUPIR.iml | 10 +
.idea/deployment.xml | 23 +
.idea/inspectionProfiles/Project_Default.xml | 38 +
.../inspectionProfiles/profiles_settings.xml | 6 +
.idea/misc.xml | 4 +
.idea/modules.xml | 8 +
.idea/vcs.xml | 6 +
.idea/webResources.xml | 14 +
CKPT_PTH.py | 4 +
LICENSE | 21 +
README.md | 125 +
SUPIR/__init__.py | 0
SUPIR/models/SUPIR_model.py | 162 +
SUPIR/models/__init__.py | 0
SUPIR/modules/SUPIR_v0.py | 718 ++
SUPIR/modules/__init__.py | 11 +
SUPIR/util.py | 173 +
SUPIR/utils/__init__.py | 0
SUPIR/utils/colorfix.py | 120 +
assets/framework.png | Bin 0 -> 976041 bytes
assets/teaser.png | Bin 0 -> 4461550 bytes
gradio_demo.py | 213 +
llava/__init__.py | 1 +
llava/constants.py | 12 +
llava/conversation.py | 381 +
llava/eval/eval_gpt_review.py | 113 +
llava/eval/eval_gpt_review_bench.py | 121 +
llava/eval/eval_gpt_review_visual.py | 118 +
llava/eval/eval_pope.py | 81 +
llava/eval/eval_science_qa.py | 114 +
llava/eval/eval_science_qa_gpt4.py | 104 +
llava/eval/eval_science_qa_gpt4_requery.py | 149 +
llava/eval/eval_textvqa.py | 65 +
.../eval/generate_webpage_data_from_table.py | 111 +
llava/eval/m4c_evaluator.py | 334 +
llava/eval/model_qa.py | 85 +
llava/eval/model_vqa.py | 125 +
llava/eval/model_vqa_loader.py | 144 +
llava/eval/model_vqa_mmbench.py | 170 +
llava/eval/model_vqa_science.py | 147 +
llava/eval/qa_baseline_gpt35.py | 74 +
llava/eval/run_llava.py | 97 +
llava/eval/summarize_gpt_review.py | 60 +
.../eval/table/answer/answer_alpaca-13b.jsonl | 80 +
llava/eval/table/answer/answer_bard.jsonl | 80 +
llava/eval/table/answer/answer_gpt35.jsonl | 80 +
.../eval/table/answer/answer_llama-13b.jsonl | 80 +
.../eval/table/answer/answer_vicuna-13b.jsonl | 80 +
.../table/caps_boxes_coco2014_val_80.jsonl | 80 +
llava/eval/table/model.jsonl | 5 +
llava/eval/table/prompt.jsonl | 4 +
llava/eval/table/question.jsonl | 80 +
.../table/results/test_sqa_llava_13b_v0.json | 8491 +++++++++++++++++
...lava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json | 8491 +++++++++++++++++
.../review/review_alpaca-13b_vicuna-13b.jsonl | 80 +
.../table/review/review_bard_vicuna-13b.jsonl | 80 +
.../review/review_gpt35_vicuna-13b.jsonl | 80 +
.../review/review_llama-13b_vicuna-13b.jsonl | 80 +
llava/eval/table/reviewer.jsonl | 4 +
llava/eval/table/rule.json | 11 +
llava/eval/webpage/figures/alpaca.png | Bin 0 -> 96061 bytes
llava/eval/webpage/figures/bard.jpg | Bin 0 -> 15309 bytes
llava/eval/webpage/figures/chatgpt.svg | 1 +
llava/eval/webpage/figures/llama.jpg | Bin 0 -> 56537 bytes
.../swords_FILL0_wght300_GRAD0_opsz48.svg | 1 +
llava/eval/webpage/figures/vicuna.jpeg | Bin 0 -> 53975 bytes
llava/eval/webpage/index.html | 162 +
llava/eval/webpage/script.js | 245 +
llava/eval/webpage/styles.css | 105 +
llava/llava_agent.py | 107 +
llava/mm_utils.py | 102 +
llava/model/__init__.py | 2 +
llava/model/apply_delta.py | 48 +
llava/model/builder.py | 148 +
llava/model/consolidate.py | 29 +
llava/model/language_model/llava_llama.py | 140 +
llava/model/language_model/llava_mpt.py | 113 +
.../language_model/mpt/adapt_tokenizer.py | 41 +
llava/model/language_model/mpt/attention.py | 300 +
llava/model/language_model/mpt/blocks.py | 41 +
.../language_model/mpt/configuration_mpt.py | 118 +
.../language_model/mpt/custom_embedding.py | 11 +
.../language_model/mpt/flash_attn_triton.py | 484 +
.../mpt/hf_prefixlm_converter.py | 415 +
.../language_model/mpt/meta_init_context.py | 94 +
.../model/language_model/mpt/modeling_mpt.py | 331 +
llava/model/language_model/mpt/norm.py | 56 +
.../language_model/mpt/param_init_fns.py | 181 +
llava/model/llava_arch.py | 256 +
llava/model/make_delta.py | 52 +
llava/model/multimodal_encoder/builder.py | 11 +
.../model/multimodal_encoder/clip_encoder.py | 81 +
llava/model/multimodal_projector/builder.py | 51 +
llava/model/utils.py | 20 +
llava/serve/__init__.py | 0
llava/serve/cli.py | 125 +
llava/serve/controller.py | 298 +
llava/serve/examples/extreme_ironing.jpg | Bin 0 -> 62587 bytes
llava/serve/examples/waterview.jpg | Bin 0 -> 95499 bytes
llava/serve/gradio_web_server.py | 420 +
llava/serve/model_worker.py | 285 +
llava/serve/register_worker.py | 26 +
llava/serve/test_message.py | 62 +
llava/train/llama_flash_attn_monkey_patch.py | 115 +
llava/train/llava_trainer.py | 175 +
llava/train/train.py | 952 ++
llava/train/train_mem.py | 13 +
llava/utils.py | 126 +
options/SUPIR_v0.yaml | 156 +
requirements.txt | 38 +
sgm/__init__.py | 4 +
sgm/lr_scheduler.py | 135 +
sgm/models/__init__.py | 2 +
sgm/models/autoencoder.py | 335 +
sgm/models/diffusion.py | 320 +
sgm/modules/__init__.py | 8 +
sgm/modules/attention.py | 635 ++
sgm/modules/autoencoding/__init__.py | 0
sgm/modules/autoencoding/losses/__init__.py | 246 +
sgm/modules/autoencoding/lpips/__init__.py | 0
.../autoencoding/lpips/loss/.gitignore | 1 +
sgm/modules/autoencoding/lpips/loss/LICENSE | 23 +
.../autoencoding/lpips/loss/__init__.py | 0
sgm/modules/autoencoding/lpips/loss/lpips.py | 147 +
sgm/modules/autoencoding/lpips/model/LICENSE | 58 +
.../autoencoding/lpips/model/__init__.py | 0
sgm/modules/autoencoding/lpips/model/model.py | 88 +
sgm/modules/autoencoding/lpips/util.py | 128 +
.../autoencoding/lpips/vqperceptual.py | 17 +
.../autoencoding/regularizers/__init__.py | 53 +
sgm/modules/diffusionmodules/__init__.py | 7 +
sgm/modules/diffusionmodules/denoiser.py | 73 +
.../diffusionmodules/denoiser_scaling.py | 31 +
.../diffusionmodules/denoiser_weighting.py | 24 +
sgm/modules/diffusionmodules/discretizer.py | 69 +
sgm/modules/diffusionmodules/guiders.py | 88 +
sgm/modules/diffusionmodules/loss.py | 69 +
sgm/modules/diffusionmodules/model.py | 743 ++
sgm/modules/diffusionmodules/openaimodel.py | 1272 +++
sgm/modules/diffusionmodules/sampling.py | 449 +
.../diffusionmodules/sampling_utils.py | 48 +
.../diffusionmodules/sigma_sampling.py | 40 +
sgm/modules/diffusionmodules/util.py | 309 +
sgm/modules/diffusionmodules/wrappers.py | 103 +
sgm/modules/distributions/__init__.py | 0
sgm/modules/distributions/distributions.py | 102 +
sgm/modules/ema.py | 86 +
sgm/modules/encoders/__init__.py | 0
sgm/modules/encoders/modules.py | 1063 +++
sgm/util.py | 248 +
test.py | 89 +
153 files changed, 35807 insertions(+)
create mode 100644 .idea/.gitignore
create mode 100644 .idea/.name
create mode 100644 .idea/SUPIR.iml
create mode 100644 .idea/deployment.xml
create mode 100644 .idea/inspectionProfiles/Project_Default.xml
create mode 100644 .idea/inspectionProfiles/profiles_settings.xml
create mode 100644 .idea/misc.xml
create mode 100644 .idea/modules.xml
create mode 100644 .idea/vcs.xml
create mode 100644 .idea/webResources.xml
create mode 100644 CKPT_PTH.py
create mode 100644 LICENSE
create mode 100644 README.md
create mode 100644 SUPIR/__init__.py
create mode 100644 SUPIR/models/SUPIR_model.py
create mode 100644 SUPIR/models/__init__.py
create mode 100644 SUPIR/modules/SUPIR_v0.py
create mode 100644 SUPIR/modules/__init__.py
create mode 100644 SUPIR/util.py
create mode 100644 SUPIR/utils/__init__.py
create mode 100644 SUPIR/utils/colorfix.py
create mode 100644 assets/framework.png
create mode 100644 assets/teaser.png
create mode 100644 gradio_demo.py
create mode 100644 llava/__init__.py
create mode 100644 llava/constants.py
create mode 100644 llava/conversation.py
create mode 100644 llava/eval/eval_gpt_review.py
create mode 100644 llava/eval/eval_gpt_review_bench.py
create mode 100644 llava/eval/eval_gpt_review_visual.py
create mode 100644 llava/eval/eval_pope.py
create mode 100644 llava/eval/eval_science_qa.py
create mode 100644 llava/eval/eval_science_qa_gpt4.py
create mode 100644 llava/eval/eval_science_qa_gpt4_requery.py
create mode 100644 llava/eval/eval_textvqa.py
create mode 100644 llava/eval/generate_webpage_data_from_table.py
create mode 100644 llava/eval/m4c_evaluator.py
create mode 100644 llava/eval/model_qa.py
create mode 100644 llava/eval/model_vqa.py
create mode 100644 llava/eval/model_vqa_loader.py
create mode 100644 llava/eval/model_vqa_mmbench.py
create mode 100644 llava/eval/model_vqa_science.py
create mode 100644 llava/eval/qa_baseline_gpt35.py
create mode 100644 llava/eval/run_llava.py
create mode 100644 llava/eval/summarize_gpt_review.py
create mode 100644 llava/eval/table/answer/answer_alpaca-13b.jsonl
create mode 100644 llava/eval/table/answer/answer_bard.jsonl
create mode 100644 llava/eval/table/answer/answer_gpt35.jsonl
create mode 100644 llava/eval/table/answer/answer_llama-13b.jsonl
create mode 100644 llava/eval/table/answer/answer_vicuna-13b.jsonl
create mode 100644 llava/eval/table/caps_boxes_coco2014_val_80.jsonl
create mode 100644 llava/eval/table/model.jsonl
create mode 100644 llava/eval/table/prompt.jsonl
create mode 100644 llava/eval/table/question.jsonl
create mode 100644 llava/eval/table/results/test_sqa_llava_13b_v0.json
create mode 100644 llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json
create mode 100644 llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl
create mode 100644 llava/eval/table/review/review_bard_vicuna-13b.jsonl
create mode 100644 llava/eval/table/review/review_gpt35_vicuna-13b.jsonl
create mode 100644 llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl
create mode 100644 llava/eval/table/reviewer.jsonl
create mode 100644 llava/eval/table/rule.json
create mode 100644 llava/eval/webpage/figures/alpaca.png
create mode 100644 llava/eval/webpage/figures/bard.jpg
create mode 100644 llava/eval/webpage/figures/chatgpt.svg
create mode 100644 llava/eval/webpage/figures/llama.jpg
create mode 100644 llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg
create mode 100644 llava/eval/webpage/figures/vicuna.jpeg
create mode 100644 llava/eval/webpage/index.html
create mode 100644 llava/eval/webpage/script.js
create mode 100644 llava/eval/webpage/styles.css
create mode 100644 llava/llava_agent.py
create mode 100644 llava/mm_utils.py
create mode 100644 llava/model/__init__.py
create mode 100644 llava/model/apply_delta.py
create mode 100644 llava/model/builder.py
create mode 100644 llava/model/consolidate.py
create mode 100644 llava/model/language_model/llava_llama.py
create mode 100644 llava/model/language_model/llava_mpt.py
create mode 100644 llava/model/language_model/mpt/adapt_tokenizer.py
create mode 100644 llava/model/language_model/mpt/attention.py
create mode 100644 llava/model/language_model/mpt/blocks.py
create mode 100644 llava/model/language_model/mpt/configuration_mpt.py
create mode 100644 llava/model/language_model/mpt/custom_embedding.py
create mode 100644 llava/model/language_model/mpt/flash_attn_triton.py
create mode 100644 llava/model/language_model/mpt/hf_prefixlm_converter.py
create mode 100644 llava/model/language_model/mpt/meta_init_context.py
create mode 100644 llava/model/language_model/mpt/modeling_mpt.py
create mode 100644 llava/model/language_model/mpt/norm.py
create mode 100644 llava/model/language_model/mpt/param_init_fns.py
create mode 100644 llava/model/llava_arch.py
create mode 100644 llava/model/make_delta.py
create mode 100644 llava/model/multimodal_encoder/builder.py
create mode 100644 llava/model/multimodal_encoder/clip_encoder.py
create mode 100644 llava/model/multimodal_projector/builder.py
create mode 100644 llava/model/utils.py
create mode 100644 llava/serve/__init__.py
create mode 100644 llava/serve/cli.py
create mode 100644 llava/serve/controller.py
create mode 100644 llava/serve/examples/extreme_ironing.jpg
create mode 100644 llava/serve/examples/waterview.jpg
create mode 100644 llava/serve/gradio_web_server.py
create mode 100644 llava/serve/model_worker.py
create mode 100644 llava/serve/register_worker.py
create mode 100644 llava/serve/test_message.py
create mode 100644 llava/train/llama_flash_attn_monkey_patch.py
create mode 100644 llava/train/llava_trainer.py
create mode 100644 llava/train/train.py
create mode 100644 llava/train/train_mem.py
create mode 100644 llava/utils.py
create mode 100644 options/SUPIR_v0.yaml
create mode 100644 requirements.txt
create mode 100644 sgm/__init__.py
create mode 100644 sgm/lr_scheduler.py
create mode 100644 sgm/models/__init__.py
create mode 100644 sgm/models/autoencoder.py
create mode 100644 sgm/models/diffusion.py
create mode 100644 sgm/modules/__init__.py
create mode 100644 sgm/modules/attention.py
create mode 100644 sgm/modules/autoencoding/__init__.py
create mode 100644 sgm/modules/autoencoding/losses/__init__.py
create mode 100644 sgm/modules/autoencoding/lpips/__init__.py
create mode 100644 sgm/modules/autoencoding/lpips/loss/.gitignore
create mode 100644 sgm/modules/autoencoding/lpips/loss/LICENSE
create mode 100644 sgm/modules/autoencoding/lpips/loss/__init__.py
create mode 100644 sgm/modules/autoencoding/lpips/loss/lpips.py
create mode 100644 sgm/modules/autoencoding/lpips/model/LICENSE
create mode 100644 sgm/modules/autoencoding/lpips/model/__init__.py
create mode 100644 sgm/modules/autoencoding/lpips/model/model.py
create mode 100644 sgm/modules/autoencoding/lpips/util.py
create mode 100644 sgm/modules/autoencoding/lpips/vqperceptual.py
create mode 100644 sgm/modules/autoencoding/regularizers/__init__.py
create mode 100644 sgm/modules/diffusionmodules/__init__.py
create mode 100644 sgm/modules/diffusionmodules/denoiser.py
create mode 100644 sgm/modules/diffusionmodules/denoiser_scaling.py
create mode 100644 sgm/modules/diffusionmodules/denoiser_weighting.py
create mode 100644 sgm/modules/diffusionmodules/discretizer.py
create mode 100644 sgm/modules/diffusionmodules/guiders.py
create mode 100644 sgm/modules/diffusionmodules/loss.py
create mode 100644 sgm/modules/diffusionmodules/model.py
create mode 100644 sgm/modules/diffusionmodules/openaimodel.py
create mode 100644 sgm/modules/diffusionmodules/sampling.py
create mode 100644 sgm/modules/diffusionmodules/sampling_utils.py
create mode 100644 sgm/modules/diffusionmodules/sigma_sampling.py
create mode 100644 sgm/modules/diffusionmodules/util.py
create mode 100644 sgm/modules/diffusionmodules/wrappers.py
create mode 100644 sgm/modules/distributions/__init__.py
create mode 100644 sgm/modules/distributions/distributions.py
create mode 100644 sgm/modules/ema.py
create mode 100644 sgm/modules/encoders/__init__.py
create mode 100644 sgm/modules/encoders/modules.py
create mode 100644 sgm/util.py
create mode 100644 test.py
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..13566b8
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
diff --git a/.idea/.name b/.idea/.name
new file mode 100644
index 0000000..0b8d3da
--- /dev/null
+++ b/.idea/.name
@@ -0,0 +1 @@
+Diff4R
\ No newline at end of file
diff --git a/.idea/SUPIR.iml b/.idea/SUPIR.iml
new file mode 100644
index 0000000..de95779
--- /dev/null
+++ b/.idea/SUPIR.iml
@@ -0,0 +1,10 @@
+
+
+> Fanghua, Yu, [Jinjin Gu](https://www.jasongt.com/), Zheyuan Li, Jinfan Hu, Xiangtao Kong, [Xintao Wang](https://xinntao.github.io/), [Jingwen He](https://scholar.google.com.hk/citations?user=GUxrycUAAAAJ), [Yu Qiao](https://scholar.google.com.hk/citations?user=gFtI-8QAAAAJ), [Chao Dong](https://scholar.google.com.hk/citations?user=OSDCB0UAAAAJ)
+> Shenzhen Institute of Advanced Technology; Shanghai AI Laboratory; University of Sydney; The Hong Kong Polytechnic University; ARC Lab, Tencent PCG; The Chinese University of Hong Kong
+
+
+
+ +
+ +--- +## 🔧 Dependencies and Installation + + +1. Clone repo + ```bash + git clone https://github.com/Fanghua-Yu/SUPIR.git + cd SUPIR + ``` + +2. Install dependent packages + ```bash + conda create -n SUPIR python=3.8 -y + conda activate SUPIR + pip install --upgrade pip + pip install -r requirements.txt + ``` + +3. Download Checkpoints +#### Dependent Models +* [SDXL CLIP Encoder-1](https://huggingface.co/openai/clip-vit-large-patch14) +* [SDXL CLIP Encoder-2](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) +* [SDXL base 1.0_0.9vae](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0_0.9vae.safetensors) +* [LLaVA CLIP](https://huggingface.co/openai/clip-vit-large-patch14-336) +* [LLaVA v1.5 13B](https://huggingface.co/liuhaotian/llava-v1.5-13b) + + +#### Models we provided: +* `SUPIR-v0Q`: (Coming Soon) Google Drive, Baidu Netdisk + + Default training settings with paper. High generalization and high image quality in most cases. + +* `SUPIR-v0F`: (Coming Soon) Google Drive, Baidu Netdisk + + Training with light degradation settings. Stage1 encoder of `SUPIR-v0F` remains more details when facing light degradations. + +4. Edit Custom Path for Checkpoints + ``` + * [CKPT_PTH.py] --> LLAVA_CLIP_PATH, LLAVA_MODEL_PATH, SDXL_CLIP1_PATH, SDXL_CLIP2_CACHE_DIR + * [options/SUPIR_v0.yaml] --> SDXL_CKPT, SUPIR_CKPT_Q, SUPIR_CKPT_F + ``` +--- + +## ⚡ Quick Inference + + +### Usage of SUPIR + +```console +Usage: +-- python test.py [options] +-- python gradio_demo.py [interactive options] + +--img_dir Input folder. +--save_dir Output folder. +--upscale Upsampling ratio of given inputs. Default: 1 +--SUPIR_sign Model selection. Default: 'Q'; Options: ['F', 'Q'] +--seed Random seed. Default: 1234 +--min_size Minimum resolution of output images. Default: 1024 +--edm_steps Numb of steps for EDM Sampling Scheduler. Default: 50 +--s_stage1 Control Strength of Stage1. Default: -1 (negative means invalid) +--s_churn Original hy-param of EDM. Default: 5 +--s_noise Original hy-param of EDM. Default: 1.003 +--s_cfg Classifier-free guidance scale for prompts. Default: 7.5 +--s_stage2 Control Strength of Stage2. Default: 1.0 +--num_samples Number of samples for each input. Default: 1 +--a_prompt Additive positive prompt for all inputs. + Default: 'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, + hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme + meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.' +--n_prompt Fixed negative prompt for all inputs. + Default: 'painting, oil painting, illustration, drawing, art, sketch, oil painting, + cartoon, CG Style, 3D render, unreal engine, blurring, dirty, messy, worst quality, + low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth' +--color_fix_type Color Fixing Type. Default: 'Wavelet'; Options: ['None', 'AdaIn', 'Wavelet'] +--linear_CFG Linearly (with sigma) increase CFG from 'spt_linear_CFG' to s_cfg. Default: False +--linear_s_stage2 Linearly (with sigma) increase s_stage2 from 'spt_linear_s_stage2' to s_stage2. Default: False +--spt_linear_CFG Start point of linearly increasing CFG. Default: 1.0 +--spt_linear_s_stage2 Start point of linearly increasing s_stage2. Default: 0.0 +--ae_dtype Inference data type of AutoEncoder. Default: 'bf16'; Options: ['fp32', 'bf16'] +--diff_dtype Inference data type of Diffusion. Default: 'fp16'; Options: ['fp32', 'fp16', 'bf16'] +``` + +### Python Script +```Shell +# Seek for best quality for most cases +CUDA_VISIBLE_DEVICES=0,1 python test.py --img_dir '/opt/data/private/LV_Dataset/DiffGLV-Test-All/RealPhoto60/LQ' --save_dir ./results-Q --SUPIR_sign Q --upscale 2 +# for light degradation and high fidelity +CUDA_VISIBLE_DEVICES=0,1 python test.py --img_dir '/opt/data/private/LV_Dataset/DiffGLV-Test-All/RealPhoto60/LQ' --save_dir ./results-F --SUPIR_sign F --upscale 2 --s_cfg 4.0 --linear_CFG +``` + +### Gradio Demo +```Shell +CUDA_VISIBLE_DEVICES=0,1 python gradio_demo.py --ip 0.0.0.0 --port 6688 +``` + +### Online Demo (Coming Soon) + + + +--- + +## BibTeX + @misc{yu2024scaling, + title={Scaling Up to Excellence: Practicing Model Scaling for Photo-Realistic Image Restoration In the Wild}, + author={Fanghua Yu and Jinjin Gu and Zheyuan Li and Jinfan Hu and Xiangtao Kong and Xintao Wang and Jingwen He and Yu Qiao and Chao Dong}, + year={2024}, + eprint={2401.13627}, + archivePrefix={arXiv}, + primaryClass={cs.CV} + } + +## 📧 Contact +If you have any question, please email `fanghuayu96@gmail.com`. diff --git a/SUPIR/__init__.py b/SUPIR/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SUPIR/models/SUPIR_model.py b/SUPIR/models/SUPIR_model.py new file mode 100644 index 0000000..25367b2 --- /dev/null +++ b/SUPIR/models/SUPIR_model.py @@ -0,0 +1,162 @@ +import torch +from sgm.models.diffusion import DiffusionEngine +from sgm.util import instantiate_from_config +import copy +from sgm.modules.distributions.distributions import DiagonalGaussianDistribution +import random +from SUPIR.utils.colorfix import wavelet_reconstruction, adaptive_instance_normalization +from pytorch_lightning import seed_everything +from torch.nn.functional import interpolate + +class SUPIRModel(DiffusionEngine): + def __init__(self, control_stage_config, ae_dtype='fp32', diffusion_dtype='fp32', p_p='', n_p='', *args, **kwargs): + super().__init__(*args, **kwargs) + control_model = instantiate_from_config(control_stage_config) + self.model.load_control_model(control_model) + self.first_stage_model.denoise_encoder = copy.deepcopy(self.first_stage_model.encoder) + self.sampler_config = kwargs['sampler_config'] + + assert (ae_dtype in ['fp32', 'fp16', 'bf16']) and (diffusion_dtype in ['fp32', 'fp16', 'bf16']) + if ae_dtype == 'fp32': + ae_dtype = torch.float32 + elif ae_dtype == 'fp16': + raise RuntimeError('fp16 cause NaN in AE') + elif ae_dtype == 'bf16': + ae_dtype = torch.bfloat16 + + if diffusion_dtype == 'fp32': + diffusion_dtype = torch.float32 + elif diffusion_dtype == 'fp16': + diffusion_dtype = torch.float16 + elif diffusion_dtype == 'bf16': + diffusion_dtype = torch.bfloat16 + + self.ae_dtype = ae_dtype + self.model.dtype = diffusion_dtype + + self.p_p = p_p + self.n_p = n_p + + @torch.no_grad() + def encode_first_stage(self, x): + with torch.autocast("cuda", dtype=self.ae_dtype): + z = self.first_stage_model.encode(x) + z = self.scale_factor * z + return z + + @torch.no_grad() + def encode_first_stage_with_denoise(self, x, use_sample=True): + with torch.autocast("cuda", dtype=self.ae_dtype): + h = self.first_stage_model.denoise_encoder(x) + moments = self.first_stage_model.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + if use_sample: + z = posterior.sample() + else: + z = posterior.mode() + z = self.scale_factor * z + return z + + @torch.no_grad() + def decode_first_stage(self, z): + z = 1.0 / self.scale_factor * z + with torch.autocast("cuda", dtype=self.ae_dtype): + out = self.first_stage_model.decode(z) + return out.float() + + @torch.no_grad() + def batchify_denoise(self, x): + ''' + [N, C, H, W], [-1, 1], RGB + ''' + x = self.encode_first_stage_with_denoise(x, use_sample=False) + return self.decode_first_stage(x) + + @torch.no_grad() + def batchify_sample(self, x, p, p_p='default', n_p='default', num_steps=100, restoration_scale=4.0, s_churn=0, s_noise=1.003, cfg_scale=4.0, seed=-1, + num_samples=1, control_scale=1, color_fix_type='None', use_linear_CFG=False, use_linear_control_scale=False, + cfg_scale_start=1.0, control_scale_start=0.0, **kwargs): + ''' + [N, C], [-1, 1], RGB + ''' + assert len(x) == len(p) + assert color_fix_type in ['Wavelet', 'AdaIn', 'None'] + + N = len(x) + if num_samples > 1: + assert N == 1 + N = num_samples + x = x.repeat(N, 1, 1, 1) + p = p * N + + if p_p == 'default': + p_p = self.p_p + if n_p == 'default': + n_p = self.n_p + + self.sampler_config.params.num_steps = num_steps + if use_linear_CFG: + self.sampler_config.params.guider_config.params.scale_min = cfg_scale + self.sampler_config.params.guider_config.params.scale = cfg_scale_start + else: + self.sampler_config.params.guider_config.params.scale = cfg_scale + self.sampler_config.params.restore_cfg = restoration_scale + self.sampler_config.params.s_churn = s_churn + self.sampler_config.params.s_noise = s_noise + self.sampler = instantiate_from_config(self.sampler_config) + + if seed == -1: + seed = random.randint(0, 65535) + seed_everything(seed) + + _z = self.encode_first_stage_with_denoise(x, use_sample=False) + + x_stage1 = self.decode_first_stage(_z) + # x_stage1 = interpolate(x_stage1, scale_factor=scale_factor, mode='bilinear', antialias=True) + # _z = self.encode_first_stage_with_denoise(x_stage1) + + z_stage1 = self.encode_first_stage(x_stage1) + + batch = {} + batch['txt'] = [''.join([_p, p_p]) for _p in p] + batch['original_size_as_tuple'] = torch.tensor([1024, 1024]).repeat(N, 1).to(x.device) + batch['crop_coords_top_left'] = torch.tensor([0, 0]).repeat(N, 1).to(x.device) + batch['target_size_as_tuple'] = torch.tensor([1024, 1024]).repeat(N, 1).to(x.device) + batch['aesthetic_score'] = torch.tensor([9.0]).repeat(N, 1).to(x.device) + batch['control'] = _z + + batch_uc = copy.deepcopy(batch) + batch_uc['txt'] = [n_p for _ in p] + + c, uc = self.conditioner.get_unconditional_conditioning(batch, batch_uc) + + denoiser = lambda input, sigma, c, control_scale: self.denoiser( + self.model, input, sigma, c, control_scale, **kwargs + ) + + noised_z = torch.randn_like(_z).to(_z.device) + + _samples = self.sampler(denoiser, noised_z, cond=c, uc=uc, x_center=z_stage1, control_scale=control_scale, + use_linear_control_scale=use_linear_control_scale, control_scale_start=control_scale_start) + samples = self.decode_first_stage(_samples) + if color_fix_type == 'Wavelet': + samples = wavelet_reconstruction(samples, x_stage1) + elif color_fix_type == 'AdaIn': + samples = adaptive_instance_normalization(samples, x_stage1) + return samples + + +if __name__ == '__main__': + from SUPIR.util import create_model, load_state_dict + + model = create_model('../../options/dev/SUPIR_paper_version.yaml') + + SDXL_CKPT = '/opt/data/private/AIGC_pretrain/SDXL_cache/sd_xl_base_1.0_0.9vae.safetensors' + SUPIR_CKPT = '/opt/data/private/AIGC_pretrain/SUPIR_cache/SUPIR-paper.ckpt' + model.load_state_dict(load_state_dict(SDXL_CKPT), strict=False) + model.load_state_dict(load_state_dict(SUPIR_CKPT), strict=False) + model = model.cuda() + + x = torch.randn(1, 3, 512, 512).cuda() + p = ['a professional, detailed, high-quality photo'] + samples = model.batchify_sample(x, p, num_steps=50, restoration_scale=4.0, s_churn=0, cfg_scale=4.0, seed=-1, num_samples=1) diff --git a/SUPIR/models/__init__.py b/SUPIR/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SUPIR/modules/SUPIR_v0.py b/SUPIR/modules/SUPIR_v0.py new file mode 100644 index 0000000..7454eb8 --- /dev/null +++ b/SUPIR/modules/SUPIR_v0.py @@ -0,0 +1,718 @@ +# from einops._torch_specific import allow_ops_in_compiled_graph +# allow_ops_in_compiled_graph() +import einops +import torch +import torch as th +import torch.nn as nn +from einops import rearrange, repeat + +from sgm.modules.diffusionmodules.util import ( + avg_pool_nd, + checkpoint, + conv_nd, + linear, + normalization, + timestep_embedding, + zero_module, +) + +from sgm.modules.diffusionmodules.openaimodel import Downsample, Upsample, UNetModel, Timestep, \ + TimestepEmbedSequential, ResBlock, AttentionBlock, TimestepBlock +from sgm.modules.attention import SpatialTransformer, MemoryEfficientCrossAttention, CrossAttention +from sgm.util import default, log_txt_as_img, exists, instantiate_from_config +import re +import torch +from functools import partial + + +try: + import xformers + import xformers.ops + XFORMERS_IS_AVAILBLE = True +except: + XFORMERS_IS_AVAILBLE = False + + +# dummy replace +def convert_module_to_f16(x): + pass + + +def convert_module_to_f32(x): + pass + + +class ZeroConv(nn.Module): + def __init__(self, label_nc, norm_nc, mask=False): + super().__init__() + self.zero_conv = zero_module(conv_nd(2, label_nc, norm_nc, 1, 1, 0)) + self.mask = mask + + def forward(self, c, h, h_ori=None): + # with torch.cuda.amp.autocast(enabled=False, dtype=torch.float32): + if not self.mask: + h = h + self.zero_conv(c) + else: + h = h + self.zero_conv(c) * torch.zeros_like(h) + if h_ori is not None: + h = th.cat([h_ori, h], dim=1) + return h + + +class ZeroSFT(nn.Module): + def __init__(self, label_nc, norm_nc, concat_channels=0, norm=True, mask=False): + super().__init__() + + # param_free_norm_type = str(parsed.group(1)) + ks = 3 + pw = ks // 2 + + self.norm = norm + if self.norm: + self.param_free_norm = normalization(norm_nc + concat_channels) + else: + self.param_free_norm = nn.Identity() + + nhidden = 128 + + self.mlp_shared = nn.Sequential( + nn.Conv2d(label_nc, nhidden, kernel_size=ks, padding=pw), + nn.SiLU() + ) + self.zero_mul = zero_module(nn.Conv2d(nhidden, norm_nc + concat_channels, kernel_size=ks, padding=pw)) + self.zero_add = zero_module(nn.Conv2d(nhidden, norm_nc + concat_channels, kernel_size=ks, padding=pw)) + # self.zero_mul = nn.Conv2d(nhidden, norm_nc + concat_channels, kernel_size=ks, padding=pw) + # self.zero_add = nn.Conv2d(nhidden, norm_nc + concat_channels, kernel_size=ks, padding=pw) + + self.zero_conv = zero_module(conv_nd(2, label_nc, norm_nc, 1, 1, 0)) + self.pre_concat = bool(concat_channels != 0) + self.mask = mask + + def forward(self, c, h, h_ori=None, control_scale=1): + assert self.mask is False + if h_ori is not None and self.pre_concat: + h_raw = th.cat([h_ori, h], dim=1) + else: + h_raw = h + + if self.mask: + h = h + self.zero_conv(c) * torch.zeros_like(h) + else: + h = h + self.zero_conv(c) + if h_ori is not None and self.pre_concat: + h = th.cat([h_ori, h], dim=1) + actv = self.mlp_shared(c) + gamma = self.zero_mul(actv) + beta = self.zero_add(actv) + if self.mask: + gamma = gamma * torch.zeros_like(gamma) + beta = beta * torch.zeros_like(beta) + h = self.param_free_norm(h) * (gamma + 1) + beta + if h_ori is not None and not self.pre_concat: + h = th.cat([h_ori, h], dim=1) + return h * control_scale + h_raw * (1 - control_scale) + + +class ZeroCrossAttn(nn.Module): + ATTENTION_MODES = { + "softmax": CrossAttention, # vanilla attention + "softmax-xformers": MemoryEfficientCrossAttention + } + + def __init__(self, context_dim, query_dim, zero_out=True, mask=False): + super().__init__() + attn_mode = "softmax-xformers" if XFORMERS_IS_AVAILBLE else "softmax" + assert attn_mode in self.ATTENTION_MODES + attn_cls = self.ATTENTION_MODES[attn_mode] + self.attn = attn_cls(query_dim=query_dim, context_dim=context_dim, heads=query_dim//64, dim_head=64) + self.norm1 = normalization(query_dim) + self.norm2 = normalization(context_dim) + + self.mask = mask + + # if zero_out: + # # for p in self.attn.to_out.parameters(): + # # p.detach().zero_() + # self.attn.to_out = zero_module(self.attn.to_out) + + def forward(self, context, x, control_scale=1): + assert self.mask is False + x_in = x + x = self.norm1(x) + context = self.norm2(context) + b, c, h, w = x.shape + x = rearrange(x, 'b c h w -> b (h w) c').contiguous() + context = rearrange(context, 'b c h w -> b (h w) c').contiguous() + x = self.attn(x, context) + x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w).contiguous() + if self.mask: + x = x * torch.zeros_like(x) + x = x_in + x * control_scale + + return x + + +class GLVControl(nn.Module): + def __init__( + self, + in_channels, + model_channels, + out_channels, + num_res_blocks, + attention_resolutions, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + num_classes=None, + use_checkpoint=False, + use_fp16=False, + num_heads=-1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + use_spatial_transformer=False, # custom transformer support + transformer_depth=1, # custom transformer support + context_dim=None, # custom transformer support + n_embed=None, # custom support for prediction of discrete ids into codebook of first stage vq model + legacy=True, + disable_self_attentions=None, + num_attention_blocks=None, + disable_middle_self_attn=False, + use_linear_in_transformer=False, + spatial_transformer_attn_type="softmax", + adm_in_channels=None, + use_fairscale_checkpoint=False, + offload_to_cpu=False, + transformer_depth_middle=None, + input_upscale=1, + ): + super().__init__() + from omegaconf.listconfig import ListConfig + + if use_spatial_transformer: + assert ( + context_dim is not None + ), "Fool!! You forgot to include the dimension of your cross-attention conditioning..." + + if context_dim is not None: + assert ( + use_spatial_transformer + ), "Fool!! You forgot to use the spatial transformer for your cross-attention conditioning..." + if type(context_dim) == ListConfig: + context_dim = list(context_dim) + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + if num_heads == -1: + assert ( + num_head_channels != -1 + ), "Either num_heads or num_head_channels has to be set" + + if num_head_channels == -1: + assert ( + num_heads != -1 + ), "Either num_heads or num_head_channels has to be set" + + self.in_channels = in_channels + self.model_channels = model_channels + self.out_channels = out_channels + if isinstance(transformer_depth, int): + transformer_depth = len(channel_mult) * [transformer_depth] + elif isinstance(transformer_depth, ListConfig): + transformer_depth = list(transformer_depth) + transformer_depth_middle = default( + transformer_depth_middle, transformer_depth[-1] + ) + + if isinstance(num_res_blocks, int): + self.num_res_blocks = len(channel_mult) * [num_res_blocks] + else: + if len(num_res_blocks) != len(channel_mult): + raise ValueError( + "provide num_res_blocks either as an int (globally constant) or " + "as a list/tuple (per-level) with the same length as channel_mult" + ) + self.num_res_blocks = num_res_blocks + # self.num_res_blocks = num_res_blocks + if disable_self_attentions is not None: + # should be a list of booleans, indicating whether to disable self-attention in TransformerBlocks or not + assert len(disable_self_attentions) == len(channel_mult) + if num_attention_blocks is not None: + assert len(num_attention_blocks) == len(self.num_res_blocks) + assert all( + map( + lambda i: self.num_res_blocks[i] >= num_attention_blocks[i], + range(len(num_attention_blocks)), + ) + ) + print( + f"Constructor of UNetModel received num_attention_blocks={num_attention_blocks}. " + f"This option has LESS priority than attention_resolutions {attention_resolutions}, " + f"i.e., in cases where num_attention_blocks[i] > 0 but 2**i not in attention_resolutions, " + f"attention will still not be set." + ) # todo: convert to warning + + self.attention_resolutions = attention_resolutions + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + if use_fp16: + print("WARNING: use_fp16 was dropped and has no effect anymore.") + # self.dtype = th.float16 if use_fp16 else th.float32 + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + self.predict_codebook_ids = n_embed is not None + + assert use_fairscale_checkpoint != use_checkpoint or not ( + use_checkpoint or use_fairscale_checkpoint + ) + + self.use_fairscale_checkpoint = False + checkpoint_wrapper_fn = ( + partial(checkpoint_wrapper, offload_to_cpu=offload_to_cpu) + if self.use_fairscale_checkpoint + else lambda x: x + ) + + time_embed_dim = model_channels * 4 + self.time_embed = checkpoint_wrapper_fn( + nn.Sequential( + linear(model_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ) + ) + + if self.num_classes is not None: + if isinstance(self.num_classes, int): + self.label_emb = nn.Embedding(num_classes, time_embed_dim) + elif self.num_classes == "continuous": + print("setting up linear c_adm embedding layer") + self.label_emb = nn.Linear(1, time_embed_dim) + elif self.num_classes == "timestep": + self.label_emb = checkpoint_wrapper_fn( + nn.Sequential( + Timestep(model_channels), + nn.Sequential( + linear(model_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ), + ) + ) + elif self.num_classes == "sequential": + assert adm_in_channels is not None + self.label_emb = nn.Sequential( + nn.Sequential( + linear(adm_in_channels, time_embed_dim), + nn.SiLU(), + linear(time_embed_dim, time_embed_dim), + ) + ) + else: + raise ValueError() + + self.input_blocks = nn.ModuleList( + [ + TimestepEmbedSequential( + conv_nd(dims, in_channels, model_channels, 3, padding=1) + ) + ] + ) + self._feature_size = model_channels + input_block_chans = [model_channels] + ch = model_channels + ds = 1 + for level, mult in enumerate(channel_mult): + for nr in range(self.num_res_blocks[level]): + layers = [ + checkpoint_wrapper_fn( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=mult * model_channels, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ) + ] + ch = mult * model_channels + if ds in attention_resolutions: + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + # num_heads = 1 + dim_head = ( + ch // num_heads + if use_spatial_transformer + else num_head_channels + ) + if exists(disable_self_attentions): + disabled_sa = disable_self_attentions[level] + else: + disabled_sa = False + + if ( + not exists(num_attention_blocks) + or nr < num_attention_blocks[level] + ): + layers.append( + checkpoint_wrapper_fn( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=dim_head, + use_new_attention_order=use_new_attention_order, + ) + ) + if not use_spatial_transformer + else checkpoint_wrapper_fn( + SpatialTransformer( + ch, + num_heads, + dim_head, + depth=transformer_depth[level], + context_dim=context_dim, + disable_self_attn=disabled_sa, + use_linear=use_linear_in_transformer, + attn_type=spatial_transformer_attn_type, + use_checkpoint=use_checkpoint, + ) + ) + ) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + checkpoint_wrapper_fn( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + ) + ) + if resblock_updown + else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch + ) + ) + ) + ch = out_ch + input_block_chans.append(ch) + ds *= 2 + self._feature_size += ch + + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + # num_heads = 1 + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + self.middle_block = TimestepEmbedSequential( + checkpoint_wrapper_fn( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ), + checkpoint_wrapper_fn( + AttentionBlock( + ch, + use_checkpoint=use_checkpoint, + num_heads=num_heads, + num_head_channels=dim_head, + use_new_attention_order=use_new_attention_order, + ) + ) + if not use_spatial_transformer + else checkpoint_wrapper_fn( + SpatialTransformer( # always uses a self-attn + ch, + num_heads, + dim_head, + depth=transformer_depth_middle, + context_dim=context_dim, + disable_self_attn=disable_middle_self_attn, + use_linear=use_linear_in_transformer, + attn_type=spatial_transformer_attn_type, + use_checkpoint=use_checkpoint, + ) + ), + checkpoint_wrapper_fn( + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + ) + ), + ) + + self.input_upscale = input_upscale + self.input_hint_block = TimestepEmbedSequential( + zero_module(conv_nd(dims, in_channels, model_channels, 3, padding=1)) + ) + + def convert_to_fp16(self): + """ + Convert the torso of the model to float16. + """ + self.input_blocks.apply(convert_module_to_f16) + self.middle_block.apply(convert_module_to_f16) + + def convert_to_fp32(self): + """ + Convert the torso of the model to float32. + """ + self.input_blocks.apply(convert_module_to_f32) + self.middle_block.apply(convert_module_to_f32) + + def forward(self, x, timesteps, xt, context=None, y=None, **kwargs): + # with torch.cuda.amp.autocast(enabled=False, dtype=torch.float32): + # x = x.to(torch.float32) + # timesteps = timesteps.to(torch.float32) + # xt = xt.to(torch.float32) + # context = context.to(torch.float32) + # y = y.to(torch.float32) + # print(x.dtype) + xt, context, y = xt.to(x.dtype), context.to(x.dtype), y.to(x.dtype) + + if self.input_upscale != 1: + x = nn.functional.interpolate(x, scale_factor=self.input_upscale, mode='bilinear', antialias=True) + assert (y is not None) == ( + self.num_classes is not None + ), "must specify y if and only if the model is class-conditional" + hs = [] + t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype) + # import pdb + # pdb.set_trace() + emb = self.time_embed(t_emb) + + if self.num_classes is not None: + assert y.shape[0] == xt.shape[0] + emb = emb + self.label_emb(y) + + guided_hint = self.input_hint_block(x, emb, context) + + # h = x.type(self.dtype) + h = xt + for module in self.input_blocks: + if guided_hint is not None: + h = module(h, emb, context) + h += guided_hint + guided_hint = None + else: + h = module(h, emb, context) + hs.append(h) + # print(module) + # print(h.shape) + h = self.middle_block(h, emb, context) + hs.append(h) + return hs + + +class LightGLVUNet(UNetModel): + def __init__(self, mode='', project_type='ZeroSFT', project_channel_scale=1, + *args, **kwargs): + super().__init__(*args, **kwargs) + if mode == 'XL-base': + cond_output_channels = [320] * 4 + [640] * 3 + [1280] * 3 + project_channels = [160] * 4 + [320] * 3 + [640] * 3 + concat_channels = [320] * 2 + [640] * 3 + [1280] * 4 + [0] + cross_attn_insert_idx = [6, 3] + self.progressive_mask_nums = [0, 3, 7, 11] + elif mode == 'XL-refine': + cond_output_channels = [384] * 4 + [768] * 3 + [1536] * 6 + project_channels = [192] * 4 + [384] * 3 + [768] * 6 + concat_channels = [384] * 2 + [768] * 3 + [1536] * 7 + [0] + cross_attn_insert_idx = [9, 6, 3] + self.progressive_mask_nums = [0, 3, 6, 10, 14] + else: + raise NotImplementedError + + project_channels = [int(c * project_channel_scale) for c in project_channels] + + self.project_modules = nn.ModuleList() + for i in range(len(cond_output_channels)): + # if i == len(cond_output_channels) - 1: + # _project_type = 'ZeroCrossAttn' + # else: + # _project_type = project_type + _project_type = project_type + if _project_type == 'ZeroSFT': + self.project_modules.append(ZeroSFT(project_channels[i], cond_output_channels[i], + concat_channels=concat_channels[i])) + elif _project_type == 'ZeroCrossAttn': + self.project_modules.append(ZeroCrossAttn(cond_output_channels[i], project_channels[i])) + else: + raise NotImplementedError + + for i in cross_attn_insert_idx: + self.project_modules.insert(i, ZeroCrossAttn(cond_output_channels[i], concat_channels[i])) + # print(self.project_modules[i]) + + def step_progressive_mask(self): + if len(self.progressive_mask_nums) > 0: + mask_num = self.progressive_mask_nums.pop() + for i in range(len(self.project_modules)): + if i < mask_num: + self.project_modules[i].mask = True + else: + self.project_modules[i].mask = False + return + # print(f'step_progressive_mask, current masked layers: {mask_num}') + else: + return + # print('step_progressive_mask, no more masked layers') + # for i in range(len(self.project_modules)): + # print(self.project_modules[i].mask) + + + def forward(self, x, timesteps=None, context=None, y=None, control=None, control_scale=1, **kwargs): + """ + Apply the model to an input batch. + :param x: an [N x C x ...] Tensor of inputs. + :param timesteps: a 1-D batch of timesteps. + :param context: conditioning plugged in via crossattn + :param y: an [N] Tensor of labels, if class-conditional. + :return: an [N x C x ...] Tensor of outputs. + """ + assert (y is not None) == ( + self.num_classes is not None + ), "must specify y if and only if the model is class-conditional" + hs = [] + + _dtype = control[0].dtype + x, context, y = x.to(_dtype), context.to(_dtype), y.to(_dtype) + + with torch.no_grad(): + t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype) + emb = self.time_embed(t_emb) + + if self.num_classes is not None: + assert y.shape[0] == x.shape[0] + emb = emb + self.label_emb(y) + + # h = x.type(self.dtype) + h = x + for module in self.input_blocks: + h = module(h, emb, context) + hs.append(h) + + adapter_idx = len(self.project_modules) - 1 + control_idx = len(control) - 1 + h = self.middle_block(h, emb, context) + h = self.project_modules[adapter_idx](control[control_idx], h, control_scale=control_scale) + adapter_idx -= 1 + control_idx -= 1 + + for i, module in enumerate(self.output_blocks): + _h = hs.pop() + h = self.project_modules[adapter_idx](control[control_idx], _h, h, control_scale=control_scale) + adapter_idx -= 1 + # h = th.cat([h, _h], dim=1) + if len(module) == 3: + assert isinstance(module[2], Upsample) + for layer in module[:2]: + if isinstance(layer, TimestepBlock): + h = layer(h, emb) + elif isinstance(layer, SpatialTransformer): + h = layer(h, context) + else: + h = layer(h) + # print('cross_attn_here') + h = self.project_modules[adapter_idx](control[control_idx], h, control_scale=control_scale) + adapter_idx -= 1 + h = module[2](h) + else: + h = module(h, emb, context) + control_idx -= 1 + # print(module) + # print(h.shape) + + h = h.type(x.dtype) + if self.predict_codebook_ids: + assert False, "not supported anymore. what the f*** are you doing?" + else: + return self.out(h) + +if __name__ == '__main__': + from omegaconf import OmegaConf + + # refiner + # opt = OmegaConf.load('../../options/train/debug_p2_xl.yaml') + # + # model = instantiate_from_config(opt.model.params.control_stage_config) + # hint = model(torch.randn([1, 4, 64, 64]), torch.randn([1]), torch.randn([1, 4, 64, 64])) + # hint = [h.cuda() for h in hint] + # print(sum(map(lambda hint: hint.numel(), model.parameters()))) + # + # unet = instantiate_from_config(opt.model.params.network_config) + # unet = unet.cuda() + # + # _output = unet(torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1]).cuda(), torch.randn([1, 77, 1280]).cuda(), + # torch.randn([1, 2560]).cuda(), hint) + # print(sum(map(lambda _output: _output.numel(), unet.parameters()))) + + # base + with torch.no_grad(): + opt = OmegaConf.load('../../options/dev/SUPIR_tmp.yaml') + + model = instantiate_from_config(opt.model.params.control_stage_config) + model = model.cuda() + + hint = model(torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1]).cuda(), torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1, 77, 2048]).cuda(), + torch.randn([1, 2816]).cuda()) + + for h in hint: + print(h.shape) + # + unet = instantiate_from_config(opt.model.params.network_config) + unet = unet.cuda() + _output = unet(torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1]).cuda(), torch.randn([1, 77, 2048]).cuda(), + torch.randn([1, 2816]).cuda(), hint) + + + # model = instantiate_from_config(opt.model.params.control_stage_config) + # model = model.cuda() + # # hint = model(torch.randn([1, 4, 64, 64]), torch.randn([1]), torch.randn([1, 4, 64, 64])) + # hint = model(torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1]).cuda(), torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1, 77, 1280]).cuda(), + # torch.randn([1, 2560]).cuda()) + # # hint = [h.cuda() for h in hint] + # + # for h in hint: + # print(h.shape) + # + # unet = instantiate_from_config(opt.model.params.network_config) + # unet = unet.cuda() + # _output = unet(torch.randn([1, 4, 64, 64]).cuda(), torch.randn([1]).cuda(), torch.randn([1, 77, 1280]).cuda(), + # torch.randn([1, 2560]).cuda(), hint) diff --git a/SUPIR/modules/__init__.py b/SUPIR/modules/__init__.py new file mode 100644 index 0000000..7161cc1 --- /dev/null +++ b/SUPIR/modules/__init__.py @@ -0,0 +1,11 @@ +SDXL_BASE_CHANNEL_DICT = { + 'cond_output_channels': [320] * 4 + [640] * 3 + [1280] * 3, + 'project_channels': [160] * 4 + [320] * 3 + [640] * 3, + 'concat_channels': [320] * 2 + [640] * 3 + [1280] * 4 + [0] +} + +SDXL_REFINE_CHANNEL_DICT = { + 'cond_output_channels': [384] * 4 + [768] * 3 + [1536] * 6, + 'project_channels': [192] * 4 + [384] * 3 + [768] * 6, + 'concat_channels': [384] * 2 + [768] * 3 + [1536] * 7 + [0] +} \ No newline at end of file diff --git a/SUPIR/util.py b/SUPIR/util.py new file mode 100644 index 0000000..753a0f0 --- /dev/null +++ b/SUPIR/util.py @@ -0,0 +1,173 @@ +import os +import torch +import numpy as np +import cv2 +from PIL import Image +from torch.nn.functional import interpolate +from omegaconf import OmegaConf +from sgm.util import instantiate_from_config + + +def get_state_dict(d): + return d.get('state_dict', d) + + +def load_state_dict(ckpt_path, location='cpu'): + _, extension = os.path.splitext(ckpt_path) + if extension.lower() == ".safetensors": + import safetensors.torch + state_dict = safetensors.torch.load_file(ckpt_path, device=location) + else: + state_dict = get_state_dict(torch.load(ckpt_path, map_location=torch.device(location))) + state_dict = get_state_dict(state_dict) + print(f'Loaded state_dict from [{ckpt_path}]') + return state_dict + + +def create_model(config_path): + config = OmegaConf.load(config_path) + model = instantiate_from_config(config.model).cpu() + print(f'Loaded model config from [{config_path}]') + return model + + +def create_SUPIR_model(config_path, SUPIR_sign=None): + config = OmegaConf.load(config_path) + model = instantiate_from_config(config.model).cpu() + print(f'Loaded model config from [{config_path}]') + if config.SDXL_CKPT is not None: + model.load_state_dict(load_state_dict(config.SDXL_CKPT), strict=False) + if config.SUPIR_CKPT is not None: + model.load_state_dict(load_state_dict(config.SUPIR_CKPT), strict=False) + if SUPIR_sign is not None: + assert SUPIR_sign in ['F', 'Q'] + if SUPIR_sign == 'F': + model.load_state_dict(load_state_dict(config.SUPIR_CKPT_F), strict=False) + elif SUPIR_sign == 'Q': + model.load_state_dict(load_state_dict(config.SUPIR_CKPT_Q), strict=False) + return model + +def load_QF_ckpt(config_path): + config = OmegaConf.load(config_path) + ckpt_F = torch.load(config.SUPIR_CKPT_F, map_location='cpu') + ckpt_Q = torch.load(config.SUPIR_CKPT_Q, map_location='cpu') + return ckpt_Q, ckpt_F + + +def PIL2Tensor(img, upsacle=1, min_size=1024): + ''' + PIL.Image -> Tensor[C, H, W], RGB, [-1, 1] + ''' + # size + w, h = img.size + w *= upsacle + h *= upsacle + w0, h0 = round(w), round(h) + if min(w, h) < min_size: + _upsacle = min_size / min(w, h) + w *= _upsacle + h *= _upsacle + else: + _upsacle = 1 + w = int(np.round(w / 64.0)) * 64 + h = int(np.round(h / 64.0)) * 64 + x = img.resize((w, h), Image.BICUBIC) + x = np.array(x).round().clip(0, 255).astype(np.uint8) + x = x / 255 * 2 - 1 + x = torch.tensor(x, dtype=torch.float32).permute(2, 0, 1) + return x, h0, w0 + + +def Tensor2PIL(x, h0, w0): + ''' + Tensor[C, H, W], RGB, [-1, 1] -> PIL.Image + ''' + x = x.unsqueeze(0) + x = interpolate(x, size=(h0, w0), mode='bicubic') + x = (x.squeeze(0).permute(1, 2, 0) * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8) + return Image.fromarray(x) + + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + + +def upscale_image(input_image, upscale, min_size=None, unit_resolution=64): + H, W, C = input_image.shape + H = float(H) + W = float(W) + H *= upscale + W *= upscale + if min_size is not None: + if min(H, W) < min_size: + _upsacle = min_size / min(W, H) + W *= _upsacle + H *= _upsacle + H = int(np.round(H / unit_resolution)) * unit_resolution + W = int(np.round(W / unit_resolution)) * unit_resolution + img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if upscale > 1 else cv2.INTER_AREA) + img = img.round().clip(0, 255).astype(np.uint8) + return img + + +def fix_resize(input_image, size=512, unit_resolution=64): + H, W, C = input_image.shape + H = float(H) + W = float(W) + upscale = size / min(H, W) + H *= upscale + W *= upscale + H = int(np.round(H / unit_resolution)) * unit_resolution + W = int(np.round(W / unit_resolution)) * unit_resolution + img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if upscale > 1 else cv2.INTER_AREA) + img = img.round().clip(0, 255).astype(np.uint8) + return img + + + +def Numpy2Tensor(img): + ''' + np.array[H, w, C] [0, 255] -> Tensor[C, H, W], RGB, [-1, 1] + ''' + # size + img = np.array(img) / 255 * 2 - 1 + img = torch.tensor(img, dtype=torch.float32).permute(2, 0, 1) + return img + + +def Tensor2Numpy(x, h0=None, w0=None): + ''' + Tensor[C, H, W], RGB, [-1, 1] -> PIL.Image + ''' + if h0 is not None and w0 is not None: + x = x.unsqueeze(0) + x = interpolate(x, size=(h0, w0), mode='bicubic') + x = x.squeeze(0) + x = (x.permute(1, 2, 0) * 127.5 + 127.5).cpu().numpy().clip(0, 255).astype(np.uint8) + return x + + +def convert_dtype(dtype_str): + if dtype_str == 'fp32': + return torch.float32 + elif dtype_str == 'fp16': + return torch.float16 + elif dtype_str == 'bf16': + return torch.bfloat16 + else: + raise NotImplementedError diff --git a/SUPIR/utils/__init__.py b/SUPIR/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SUPIR/utils/colorfix.py b/SUPIR/utils/colorfix.py new file mode 100644 index 0000000..32f2537 --- /dev/null +++ b/SUPIR/utils/colorfix.py @@ -0,0 +1,120 @@ +''' +# -------------------------------------------------------------------------------- +# Color fixed script from Li Yi (https://github.com/pkuliyi2015/sd-webui-stablesr/blob/master/srmodule/colorfix.py) +# -------------------------------------------------------------------------------- +''' + +import torch +from PIL import Image +from torch import Tensor +from torch.nn import functional as F + +from torchvision.transforms import ToTensor, ToPILImage + +def adain_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply adaptive instance normalization + result_tensor = adaptive_instance_normalization(target_tensor, source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + +def wavelet_color_fix(target: Image, source: Image): + # Convert images to tensors + to_tensor = ToTensor() + target_tensor = to_tensor(target).unsqueeze(0) + source_tensor = to_tensor(source).unsqueeze(0) + + # Apply wavelet reconstruction + result_tensor = wavelet_reconstruction(target_tensor, source_tensor) + + # Convert tensor back to image + to_image = ToPILImage() + result_image = to_image(result_tensor.squeeze(0).clamp_(0.0, 1.0)) + + return result_image + +def calc_mean_std(feat: Tensor, eps=1e-5): + """Calculate mean and std for adaptive_instance_normalization. + Args: + feat (Tensor): 4D tensor. + eps (float): A small value added to the variance to avoid + divide-by-zero. Default: 1e-5. + """ + size = feat.size() + assert len(size) == 4, 'The input feature should be 4D tensor.' + b, c = size[:2] + feat_var = feat.reshape(b, c, -1).var(dim=2) + eps + feat_std = feat_var.sqrt().reshape(b, c, 1, 1) + feat_mean = feat.reshape(b, c, -1).mean(dim=2).reshape(b, c, 1, 1) + return feat_mean, feat_std + +def adaptive_instance_normalization(content_feat:Tensor, style_feat:Tensor): + """Adaptive instance normalization. + Adjust the reference features to have the similar color and illuminations + as those in the degradate features. + Args: + content_feat (Tensor): The reference feature. + style_feat (Tensor): The degradate features. + """ + size = content_feat.size() + style_mean, style_std = calc_mean_std(style_feat) + content_mean, content_std = calc_mean_std(content_feat) + normalized_feat = (content_feat - content_mean.expand(size)) / content_std.expand(size) + return normalized_feat * style_std.expand(size) + style_mean.expand(size) + +def wavelet_blur(image: Tensor, radius: int): + """ + Apply wavelet blur to the input tensor. + """ + # input shape: (1, 3, H, W) + # convolution kernel + kernel_vals = [ + [0.0625, 0.125, 0.0625], + [0.125, 0.25, 0.125], + [0.0625, 0.125, 0.0625], + ] + kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device) + # add channel dimensions to the kernel to make it a 4D tensor + kernel = kernel[None, None] + # repeat the kernel across all input channels + kernel = kernel.repeat(3, 1, 1, 1) + image = F.pad(image, (radius, radius, radius, radius), mode='replicate') + # apply convolution + output = F.conv2d(image, kernel, groups=3, dilation=radius) + return output + +def wavelet_decomposition(image: Tensor, levels=5): + """ + Apply wavelet decomposition to the input tensor. + This function only returns the low frequency & the high frequency. + """ + high_freq = torch.zeros_like(image) + for i in range(levels): + radius = 2 ** i + low_freq = wavelet_blur(image, radius) + high_freq += (image - low_freq) + image = low_freq + + return high_freq, low_freq + +def wavelet_reconstruction(content_feat:Tensor, style_feat:Tensor): + """ + Apply wavelet decomposition, so that the content will have the same color as the style. + """ + # calculate the wavelet decomposition of the content feature + content_high_freq, content_low_freq = wavelet_decomposition(content_feat) + del content_low_freq + # calculate the wavelet decomposition of the style feature + style_high_freq, style_low_freq = wavelet_decomposition(style_feat) + del style_high_freq + # reconstruct the content feature with the style's high frequency + return content_high_freq + style_low_freq + diff --git a/assets/framework.png b/assets/framework.png new file mode 100644 index 0000000000000000000000000000000000000000..002358435547026d425a4442eadf9c3e4452c8e9 GIT binary patch literal 976041 zcmafa1yEe+)-4t!xI?hu?iQTjPH-C627+tjZowtE2M_KJAy{yC8VK(0`j|Up=2rdh ze~+pL`pDPk?0xoHYfFFmq#%j>_T5_u2nb|pDKTXT2+R!#2$%qPsFy9{Sp~me{=k@h zl>Z0;Q5B8&VE6_C;w`I%sOTqYQPKCG9BhFWR;CaTQeQF>L8>bAI3c5L+-;xY(89is zS(+J0L!frh2HSsUL+HEamAqADKW%;h``9N4XuMSvBuOtopuZ*)O*uW zP}1KXirbX@%czC2v B3@bCPOZ z4O6iUYz?!$;HlOg7VtniDEmUNhU*tDO0S7-Dqm=6Tg#{s1=G^-(D1tGAZwF_!caOZ z+w7`d-)=o=3-HLoQF3Ou*(QZjQEBQ@TwUJbou{{pn@rO@wVtQTJn^e4q3{3Vy#qjy zdc|IQ=P~BwHMGCAVfXFYF3H`K>-PN0b%o8tq|b>}_`e`My)iQkg2WrIgXgQZ$pHtj zRGzU++W3`4y`w|6MK)~^wT5NuFOf!}Xf1S-*erAPucVu=-myD1M+B0-^9W#K+%I{l z77*yvIPAZT2Lf$ycvFu)GrN8MHiPam7$vF?dcRNAtZb$%fnQpIUe1cJ`fi=T^zr)i zGoazPfyxqhx<4-^$Ur(pY(!7~GtETx&>($TPBXo`Ha?vj_q!?q;SCJ;%W+@Brh1*3 zx|o5Hvgz|d^Nrr{L9-d2LLJim)VkpFWK&xdL^%mdB_(O~H71p8D*zYiZCygC?4ZSv zE})@SiLW_F3x?(0M+yM0jhugJ(`*vv9xlx2m&H@2q2Xk S#C<| z8E6OdD(obCDX}0O6jeemtqVRL@)&UN1@;QFeQ!1KCB*};iX+z&CcT9d$PPT%!f~|T z{YZ^fM-L>PqS6hpVLH-0uwnkfiT{270u9z9a-bQykUOh6Am&!DN&j3%5z*${f6V z5CF}*FgH+b`4NqyegxP6dx2|74ZzKq0@TdRGEQr%9e )vBf5xw8{Fn(a%rksIcO79d$A!7zSqt1Fcq3!@)C zn@q-~pZr<87sPsZ&9m+TsJ3BqQh{eS;Am_?KRQB@@Z+X!W|1AR4nu?Er(F^szdU0B z-czB@oudvir$~d#lrxh`(?7p;#5sLyeszY(t$y^H3N`W)9#a=yrda%ueWDjFg_=>X z+{o;DJTpV`oHUo5Vh9|4{*u*Ill5W$+dmnxecjGLv)TkO;h6XK>1||*YOmCXA08a* zjBHNWopb haUl(F%$L0UyEA=E!)>?M-(zV$6^cnwvu`Rbf-my z^g+(mHp>cH5#%=9u*M@GaaB`MPTzF1G(_&Njbk=))^)PrZC({Eiz ~GH$UYaIapDU~wz-nSRyS4x 5^I2)zTb|us}mJYNJf_B$uX O3{THfz9M=lyF)4ZcR67M ze`Av5u5VE>bNC>L (<_xF!-;uOYuQBW{AqMs+c`+a z;J2u}hC9d2>eV)J&Prk$R{`!Jx#hFNc9o?r# 6j`0MnZ0LYZPqh*SAAKRX ?`iA!V?xXp)nbw`^(k~j!q00QGR?d&pyt*f@56{Ou1AR{6aTXlgpsU5Z#E| 8;H(=w#(;C) >00(B{e zd-`mHVB9xI8cQ3N#?E#;Z#Pr+D8-K*Ukyqo`D}0z$?PLa5`puFH{qi;xd+zY?qY94 zwtb=fE4^{->$QE-z3jTJ@F}?!dVT<4lm7!Ev6MAmekg}@QnT|=FK&c>;~M^2tsN(_ z{}qgPhA>K(w@vju@YA4uu^oGIK$-qdS>t}xjj=1eTRP70-0A5>sUFhM=tZTodm+UH zB_;>kH6hix`u#T20_VHZa86bGiv6v@Eld-oJfzJi)98lgAO~CL7PUYNWE8w@os7fr zJ+G;V$j@bJD0_`{d>l<7rH9iCp=Z3Y6rR2~qR*sf8GFGt3v5~hTw@VptM1?IQG^w| znH?O;vHOzbHHWs(+uFcwhZ%=uWrd2F{NAq5zaQ^Wk5)WxRy()0Ccznp;5P8B=W?I{ zDF>?yw_Yb{48xIQiC?sikQz~bvx}(px0;50M+gKq@sAY^R}!s@<&)7DCX%plahJL? zcrzN+uXU27Bx6?R=zNSnIIV$m+x!ip{M@GFFAD6oz`#-T*~u#6Ht5QH&J_yM5cE;U zM)-5^n#}2y_Hhb+byTu<1Tds|`^U7&BhlzRQ}@?9+~zb(Nf4JT_5?o%vhoFx4$Hj6 zSjKMHn-H0OUpijzZ8EX4E=8A}(ABa0c8 o#r=jux_4dpC3ii$bwVm9QGGh_U?Xu$4Q zm^&!W$RgZ=1g0OS>;q%mB7GOSqUJ-LY#;r$D#sM8`@ ^K z*=&s`1wq5`n_7
lID+-L_+PGpNtZJ1!H+J zk#Ogbr&6*zPt7x&t%njU&NF%$T3&!*?F;ML-h!Ziwcp_ mh%1cH{B>s=!4jAI$ZqK!4}FxOzZpzDA{eID%E1ye(qY};gA zQw5uIStM6f7o*>R&bv=KYWu%TyYM2mq_;R~unHH1%ISxDk*3zn$?@xR+er3t5i7Vd z7^h^3_@y{(c@H_hwfPGpQhLuc&uYtz$P@h1F*NX`ive1_iwo~DV$C`}K0G|K5psJl zw{*1h@US!kfoy$zsu~*L5!v}2A5#>4uGqtG>BE!sJfp6LE;T1P(|DkUE%KVkzn|Mr z^F)yD-Q*W&)|oXl@C9T9^*9`U#itpKw|I()%LRYSlZWWNVdv>m&v=N+=ev~Az-S1m z(C?X6t9(uK*Wh=c$6gVMG%Dk-euR=I7+m3{)KRYI^mj90jo;rat43yqlby-xKKhY; z^zqkdrkeDhvjQ8HvrA%-O=ijHtIs~wboY+U3w(}Za5uBa17~fCMq+Z|SEU7r$f8!F zee;1stVA0d2!Jr`Y#!15DJw_2sL%qUGFOuu9nslI9Qb+wply;?h!8Zd2&edQ1;dXN z)LOZulJUJkK+&>4$HstOgX~RL@1%S#fruLw3Hz@|d7@sD&V|M;rxj%x^ICKI7U#;V zILm|ic3pe- 1Ra%}q^@8{=Jl z(MRC6n!2{8hL#4QEbenRcnT<_*>c^*v=^U_3Z*T&!Ol)zt_mV{5+ZdjO}a|MA`;^& zJhx(o$~KjEJPG<$3w|15M)KP_q=U+hM8fO0(PDx_E~KJ=cF7>zrH%5K{}V7cAx{@I zg4YbMY_PTw+Tl#hYNVl~^T$Kgr$NV-N6#X*#*W9opGF_P>_7h`;fas?hd=B&X~$y@ zam1h#VD6r@o7ahU8Q+--k|whSO~e`Zmtdl|1Yj*vhyr&fh)6G@)*HpQg}<`4Trwj1 zmSKqV*%#XY_)Hb-%jABcL32-<{4}C5(?$=Njf_lS+1x2lQ0K#gmOKIh`7)PxU_*}1 zv#apHn-;1 ml!c=knYQX3t?Hzg1I%BoHwaL%Lrfk#u< zJzz2?_0RUr>?wm0-@Yi)B;hdvtvaf#bozE&WeaS^*XOq=K-WlCUEfs9t<0LXNT%V# zzAuiiS+XV+?>1&2<)kp U=KOGiwW5f8zuc>M;55b7Vb8OR-G5+ks?(*S zu1G|UO@(u4wp|*^J9f8P=6l+3-^}HMtSyCUP&e?X-PV;shZzbL=8fST7!$~5|2i%Q z4oGV@72F?NxSMCA+Gwkhv}B#ZK@z7LKC>nbiz<6cL*CNcwZ5=!*gl%*w_$JP-^>6H z_ZC1HyA9no?W91Bqm1}9ih4B-iFqI+b%Cr;R1K%5hJ(4_4fwg;t&Nf`8 x@_ff(qsr!ShQC?ykpAU_O$SrpME+kZbNN z!R2`{a=3VW>+HZI7c>J5EbNdc*CNY}l`$+Rz!t_fqGn#=!a33+R;#(*DAcAxmG_L2 zP?+5{Vx19DNT2L> Rj^Lb zKeJsTZ~O C&yST+?+X;;4M$!&pA`Q|~QlKDjC* z 1Jb*CQmvM87Bkv~fvHM5NRw7TT|kgaA4ifOkd%dC>z=O#bnN+Xq|Lo@ zb7je`|2zYC-1fNolyP`i)?*}Si<1Kgq~h@UxxBo*wcdAqywv7+-zAAkVr6Z8bbMR` zs6>!JG>QB1u|^p0W4Gph72vJX_sHjP@vV5`@WNO(6N~X!91+f}oK$#qWW|gmHsf_V z+j~ZMdVRxaBllMwTuywKUGo>s|0Q?Nh4ab(%mS!vjdg?(-&ua|PRV^ut0hmC=o^9Q z(fmEVT+b)h9VhD@PJ812s 57)yMKtZGic?DBPliXXm z9NxX?v+@^xw&dk<`0f_~o~y(mpyZaq0$w6{c+cH&qAEqw4A*>mD`wplknwQAA!Rhw z5NXM+J5Q+Hm!B7l9m!h>;t6|g+ONqtG>*~_{@ZM!BIMQA(E0S*YYB%VSbU218U3$b zSLmRuqBmzPZDGhn*IwoPgh?x;@Im%E^lVG1Io(Vu{gu1%HctrPLS9l?w=a+9m?EUW zTpk*KShR*W;=&w1X?~pIN0r_0{ l)rKfYwU|?15BB zSp}O#3PL;4<^VZ0RV{4e+m;L?mHb@ve1VA+|DgJ$e3db6bGXdSTk*3PL(;&TKB z3PhQEC1LNFgs4GUN=vC5Gv8wjYeAZ_`G+0$+ju1e3!Y%K)W_SAt@wM;aqIa^TF29V z1=xK*5)O2HXnkC15ph}RsLY7Z?v^5IBh8cw1c9z!fC_Zv>3O`|HauK-{@6yvVU-#M z5ssytR<4$fB2&N$BPumW=8N=8v3ucTVnbY#g7dRjZ9K*h-uL-pO2gipT?L>sx19U5 zwDO 7q6LFDAW>C2B}VNg>JU zwjraICvH9TEZlIBD2x}@FxxL_u>tF7C#UqJ%bgQmhm~`N?{rZf<8r0-yw_t#ls}-I zk}e )LI(|gugAs32rh+%f)w6Fs3^4_nAK%M53paZA(NUqqwyb?N`7e? zr##`-NUnc~tJ-z@WD><$?~Z`HlOZP-J#F-4t;7^xA?pM(JAP8#uul4>2b s8fQToZ9pyM_Dy5l;VgtoLtRZcxmA+Cg%-;7`7d`MZjo;$*9WUj1z zRcTzbv9R-(MMkF2y23=w;a5`OGR*j?JtH$$k1Zg iwC0M(z+E1UIZ`BJuo{)`XPy_N>PqQ?3;f;!A2Z76|cPP{+~N+H^;d;rb3ejm5Z{ zT~fuU6CD(P@)-(ns7;hS=o>2<;>6RB7V#PA0>ioxWVRlA>SHiLtewI2C0*&(L}89n z8{8=8u To)YN!a!@D;4a`Q{e2NCOKC zH7Wy|>ek^(e{cVxkM|)rn&&*M&iY=L)60podLD;zbUf7NBHgs?It~SD)ZN&^&rRrS zV>~qwM-Odr1-vD-Q#+%<|E=Uqx@4TlDYOcy4>tOQT;oMq7Quk73!H`T_hr{QP^x z>+_p``6Ud^(cD5z(WtbnnrpQNW6A)Y)gul#WKp7vfVM+uOL@dD9WB0dxRX-3lnM$^ zGk)0W)>n$-gO{W{O>KG;*ETgMD)HvpX;qJsRt$6&&EGi`UZK68z3ClQ#koI&yMSl3 zVYf)iYR`R?wz)Y&8blx}H%jy@i{-n<$~yt|OW@ipTwH88wq^S6^Z*WIHBF rM1QBP_3x ^_tpHiG*0Bqz?ya2b ij~c^f8ou`-0i? zMGmY@ ?( z`Qe-^tuHbQo-)b3Ej)0wF(=~Qhykm^G6{{(XX4*3!c!6Z6V RsZ3cT{alu8(@+GxA$|?8cOU3)cXiAtJq$*6BbB5 kIZ}G;%o=dGW;z*Jm3|i@qXcxO-8+&Q|GFzgYw31;2NgM@n-5#b+0gg1FHFtF5T5 zq_TJ6pp^gKFJ;a4pXNNNO{EfBN>h$wZJwpgT!jm$IrCY^Lauhl!?tZ#TMf@6kBcP7 z$5sRZ+oq6kzA46CUR&%9^H$L8Lrah$eSG$(cx1QwcoBZXt!~w~8=V37e0(k8KpVd* zYi6ZIT8Ae0AkuzeL?n~dInS%!uL*`*NlatPTTX)O({+^_Mpn2={B?P1nU9rUPm?G& zk6uMGCC^S_%B2#JwETnem%mh*vE;KG_L}BcMM)Ad<$s0&*ShBxYt8gNT#Q?(`!vQx z9zM*W^4}l(JdU;f1AihBzIfv)j1%qe5(5P3`!G0+GLEH)2E3)TRhVdh7*HC`QhC;G zjHpGH!(&H&$> bt@}E6vkI){`LW((S*Rn()zo#ClE|r zXs$@hM<7av3Hx3kbjoKk%&fMsGAbovIibApi9vmP=&HU?38a#gR5SAVkRdMNzFO=6 zMT?a^*X_2cdGU@zOU}z$wD8=UMK-q4`Kfdhzh47k5C_f!;Rn*8!=U?7UhmLuaS_;l z0tsj9ybTPNL_KbM9^pG~y}TQlNvvZ15j*oal_@#)C~v%?i8K2Rw`1U{2}Il{* tsYF~=notreK*H5xl!`2U&??}z4zoItq;Br?l2fcodTSz z`3V?%iy0rtwL3G#PfY6!oz(A%Z_p?5{|tJ;+pERMsIsRci~GEQ-Upbw(UK1??I+XB zkv1K-*lv(sSmhp7F@KN4#FH&~cL_afqk4S+_5v?$!chg^cHB)qEqbqx5&Z*XNy%9^ z36hsxBoC58_UZ{b3=m=orPLnk<$#FPiy4)!#^2nOIX&Kc9@3504JM3b$;sfv;i08r zbGZ?tj)n3I=|C fP1_sA&3QHJm@5lAFhbm0}}9TAx{EB(X%<=8R*b*!&>} z$Mi=8@;4rF7MyBI6C)o>(>Dv>xv)t{BaxX+vdmdld3{!u*D)RaiHcFR_ zB$7$EWNIk^hmxVxc#p~0X=(Zvx_^9gt Njj2-g7l_6~Nc9;&nfB zf4sD?yudNx$imd=z$uo`)yM;c*JQy%W=HJIAoZn?3Kn*Z3ynsyifb7h b~Q>^2pz`d7apbGJW420UfbGoif>ASmB#`_PlLhQHL6=hla3Vw@xE zFXM%-9_q{<8y;#x4*uy)HDX<~E>Ye~&Fdm4V^Jfo`{yH{1MuT00G75qZ)=f*(s3A- z9QYh1n3NwVKg&CGCyP!0h`|!#yON^}RxeIsWphn9oY%mOC0s4`T%K8_pU=cwIfwXN zIn6qWvLa)eWW8jE%glE%P$J`0{8GQ*%2N!SjiGu%rmX%h(jckboUV(mWl$vlW4YfIJ{RZyZYc z8v-Q!X|tus9C&eYc|P-almOP4MK7E>L%6LT&}Kyn9;B=H(E3RnT~qC_Ua8VHF30`; z8MDIEw~QJ!Ii(>Elvc;5`9PC+e XE=knU`^~wJ`RC`@#=^zj$P9N2o%C zkPq(C8N1xW?=?~|O^$tTa=E=vs1;iY7OwOkjF}%A|1`f|25o|zTbDLN24{5@h(dVV zMF;$mfL$<)20&lj^~#dU=@?+0j7F)B`jG{86+w&rAhGGna#^HWqR)4|{^rp3$$2-2 z{0eP(Wrpgsh9+LcM&_NwUKFM^ademBVE}#0xNc (*M%&0PiW )bSAzCOD-q88`MwF_L{bu lg$UcxP9kjei JcjgT8;JOpa_Ta??k5iI1kwne4t&by@Ugf6gl@V zG!aZ7!T1=Zs6ha1@DHAM>@~tKDXHUm|Irh1TF|&u 7Yrk77zBPG58h<(w-LQ$5`KfEyoqgzSf!vX Ig1!Mi%jB$ym{##*n%5@r95%tO|7}Hxsq_RmS4%E-RFM^N9i9@qN38vZ9~E9 zHhrB|G|5e}I_%S1{qDx)#|MoUND^Hf!Bgs-Ei>c4n{6?QNQ2i41|+BO+>eqLa)ToK zp9c1(&@5$x>OYRkj_{PguhfO|yiBJA )G;&AN5>LYNRa8SJ=N z9*uq+*M#FsjcxJZLp@r4Tm^%cz3zvpQHcX9X}FlQHWvxYeuiB@n`e Q!#U6(#xuFfMy?w3d8E_MeNAhM^D7xzC> z(~s9ADlUFncMwO4CTdAHP;WWnG*&q3j_;8V#y3)C!pFloe<|B|pDcSEFS)PxcfncV zz`8cstksbO&G5H267tiR6K6reH;p9x4g6U Glv>R+-SZ6qz3MN>O}apvmt>5hF$`7REk1pngw+Q{v(Yq&Gy^maee ztS&JZP6f3l4=ER=o2XQ~wz$bn_A|e95BYkDEV0v`<(r19&+wx9v$2!jtYqU%+|Ef> zU$h6)Bf?t_biNd+>mGluCXCcBiZDwCz~`hK+X{dmPTH=Q-S#7?QIA4^et26EWP1&w zWWGAF vnlw|-IBHoz^L z6T;{4&s>ry)W~H}=_AB+gY^*{j4Vz`inrhV37x@uWo;&4PJQI0Zm-wm|KC>Dky7K) z=Ia2E0vi8$>wMSa^Cq>8=cYlm&D;WyPISxjbce(G%2R(A(VwAP5kexjrlhoTSZVeP zV khgP~VVws<;#ps~VFm@*5HpRth=HQ7X z{fb#i@ya^5S-h_Rf+9=d!Vj<{Vp9cXcaxu^*c$Ak;~w5}2UppFmw mAIyMNr2&3w}OXrWFmc~I-C~koM!nS3ue}tS)_VXUH8qw5q*XuL#nl*lueV>|Z_^J1d zQpnuoVl<4r$Fb6TLZOXvyuW*R)| e$)|3g68<-<{_8)D7-r8)*+Ed_Z9+ws0%cNg zF>M DmC|d7nkfO(`6rzmCqA~b{e+v8W%8yPFNf|HB7Lmi_ &TV1(3TK76g2O1Vt`2}-)1T()kIrc{g ze;h6=YXRAIIPRB43%Y^?R~+Na6(P@lEVY{LK?Pgw6_g#ld@)KlPa`9R$I!&wA3At7 zbbu4u!i2-+|1kBIQEe^W({P|zfl{ovJH@4VaVf=0p}4y{1T8Mb-JJrZxVyAiad(0Q zCy?MSZ|?2={hyb0*7=gPLXxx3?3vk fuc*V9iEaOZk!I8gPWk;|d3CIV|8e7j zazUCYY(H5Le3Wz1z-s+W y!DU+7s-cpR5({;jz{BN^+xO2s-h}YJTQ$xZSYSTFW_12p*87HM zb(2h|9a002zZ=9F26AxyCqILnfWm7__v`-NO4|G#*S7Plv9s2u?f#qVt*5BZ52_?M z;6m4HXy O7{J$uPv)9zWOF?5LMSUc1jO _# zf*o|etuz#((bMD}vy7x84NNH-$b-bhhwYv11LC#>QJ~7odmyyydR~{Z^Z9Ci*EH&< zz2471Vq3M&wu)oLg7H<2+YpAR1 oFPPIlekyRFR z%Suasyk40rEvlhAWO|wcD}lIma$_gF`Z;4e?v@8z%jSkn?;OzC_lz-cRbErsBBOzwR^}7SdAVdvn?;x7m7f zFmc50I~Cu+cyH{()%|r#xOq1#3MlsXdC;h(r46dIB(7hR6b-c{%)irBIU<{rbX6%3 z`Tk|zVFyx0T^92*e(PO)YRL3HmvVXyd1Wp7kv*gJeq*d{TWXMts#xF}MUp !kwwhbQmx@kQ6PjN5?yVozu)Xxv_Q&s#5m^tjQM-Y*9dN~Eu{LG! zw_LRzYs1nb56uc_I#4L}M>a&-r(4nMtoyxu6O)<{w`Tu$_XeN&M*$N9E-o*MWseey zO!o%e@Ez>s*%92B2E~V(eisfTjPpl$G_f6CJoi_X*Xx~)w-;Q5F+Kz?I4ZgsS}Y~( z0D*LOZGO)>_xBbD1YZ0ULNDa^DyTsKz9_44ltd#csiu~p8~PWDg!)*`<_5MFVe`+x zo-qqh60eB4700;u=YdyOwNVXrq>o!pvrFM@+IzaM!KWiC+5buDx311Sp;&KoJMRw5 z87rfg1y5Y(_wu)dB5#0`)`B5lJR=OSK%jz!=>p+SamZU2p(5cOcb5V|YPII(a|z4K z$uBwM2kqDxM NBZX+bkyORbsJeDC(W{IG=I?^6@*j8ZJmR_Y2EygA?1>=M vAmAN{txsyJGHbYK}RcZb)P}QkG=Qn zn7S*2ld#`SPf6?k7ak!kEh4@r`_QMx_uccHp+l@e$Ls>#9)5cR3Mx(4<89*qAxpqN z&SdY3HX(f}HKFYPvH
w92@V{znKs!Q0`1^Yz2f&HOJvu25d zfqlQJ60P-w6v-f`6klYGnbq$n=y-2<+?x0KnBo1Lqwl@0|NT^O` Ckjn(Uf3#dfYaSLc5zb_cP{422XVUB!x;a$ zdsV~#`QNA|{6gx{#ezSPTRqPl>|Y2LwgL-hT>6%T`?N(v4ZH^p=H1Op1^(??ndi6D zz`m&c$-!J@!qWPG`WA)vOLFj{T2{!& riP)OmeU?WXj zy1K7}po!N%qqy-~K!5*#TL6@?q0rR(cCqUumh1JW8AH(zfT6Rh>)&I~5YJ^?Frv%e zTJUQMn8>1SCUj62NGiIdePjnyCS&}!@AlD47*q*(3?#{qX;`fOMts{_e?NO-JU&Zq z)qoc*AEU-xZr^6Ha=}dt%szNn*^#YuZh7tV97{2Rjr tOEh(^wfKa5x05*0~6->9FnZvdPTIT>a3*@(dck&!xs_^QxHZ=f>H!k-;^$ z9IYK_LGLwrx;RZ*q5SFufq!*g?5jV}mM6q0#7Etzs7$y2&0fi?T>3S$Gen0TOwQ3! zatF~YL9*N}FRrM9c4&P}#yu-}hMaU1u5(+}M<|f1MveT-Fc*Tf^@^3-1o+HpT#Cf& zR(s_9%OAD{FJg&=!YHaSEhWpuLt4c95k~br&NZ=;Zo2>URNsNC(A{7tXCu}iF_!&K z?I5>fscY&)-?GpIWLVkRt+Yd#op*00Nc3(yuff(^nM82vO6AAzne5L0F1o1Tv+@%c zpNk|}wIK6r9v8Qw%a*r-2IKa|jN#m#6N3MX^s@7;>Ndb5U+n8%&m7NkcBv&6Wbrv- z*K)bfQ{B$d@S$6k=h^m9_2aVb>s+XvBdqP-QuOhgiXXUm{RY&%nN;0jVxcs1#NQ#> z^8Et_E2U=R*w|QB^P2yK=lOVNIQ2<=-t9IENB7C~(*VpHt~8Xr1pX>-GURV9V0@;b zwk?%R86ewR=OX`&KXpHJxi+S|w$>}Wonk!5Bdcv|jWDO#!*#JhBSI?8UF?ONTKGNq z5Niv9L&o8A)l;(Uv?`B?$>ZAc-LNWl5nbz%#H3}mrJ7}XyEp@dk#Ac>wI7LN=D8q( zqCYJ2$7d^>ri6y-nLS%v6+vi5UK@XS#QKv_bpvY&tSFj!SZ)2^$^n0($N{U`2`xcd zZPBV6!-vleldOxp?eL72=-tWg*n-&IMrY1(DFHkg6P%@}{{r(=rYQHCFKZ>xGn(4s z)Z49WY=dePO+dtE4T}eZI(@> EQE zbjLB C~)Z7LB{UIP1X z+zOLCGHovvwGIr{;6wY@&a=+W=PZU|VjlbZ>ST9|=R8%H+6L!=12>`7Uk{khz1$pp zZ%ON%I8guNzlpPX#Ck+|YrmrMa+ZP5X-~_i9u!R;z12?*&rOs9YZjpbj3VAQH{)1W zUFXdrYqqOL#c6q79oHLckAtf7`bmr`e3Y ERoFEmJATDFoP8WrxfkG=3S zLyAj~dw@h&n4hcCVLCv^YaGlYB$z*SpoACA=jRq;wK8t>YG&et9GE;i%c7#w&7~Y^ zw)g~aXgo{ZdM6AEqd6B?QyQ;i$d=Ks%u&m~bdRDFDsg6KtGbxdR+hVG=)T);K$Apw z#mX6Jp59rBG~$XQ%HjLKgAz6w((emTGW~i)b?o#z@4a4MT3)8h%J_x%;a4lli_bbv z&PK*ueFWl|o9}33j!E8ob*Zk#c(q%cs^cMW@&MPVVlFazS0`r%5Q_HJ?gDmmJ)*)} zivw1&0U~~PXS~Rnq-+7>p2RXZM2dy)zcHtDI@le%|E9O_7=Dpj`JeOBg2`G-3swuC z{4oy?L#f0JXYWm}GbzqBJ 8zhGjc)^(UeI(+NGuxoyMGE|A&hk{`G*7u;xDSMejD@jeOeDAiK zK|yc$W;Qv}3!P&3n}_oe$+1=0$ll4KPxhhJqqW}WLv>oqF6F*2Mh_#)|BhW_l|8SY zWAc{XHs3y96sZY2KK_C2{`S5vUBOv=QV<&;_lwrm BDAIF+S7hA8ajK}O~?eu&E_HrH} zj$dz19R1JXG3BZcy?@O_@b}wqhF-iKIqs@^LF@N9^_#~PQEd;HO8O08+dUYe!>+0S zR6wXpR7sVL*Tm6}Nbm j@6H-R2zI(0&>dEKTy|bZ^HWV^ z4UX!TSWdQP5P#V20`Iz!O)~?u9cmNBUl8Y^>l?Qi`(TW+xjFWIPyP_Ae7|fNR1gAD zp1_VzR5AshONhYzddh6A4|q{5V`~L^(C=!LY84J-A*7&mMNrE!LV)-+*>+OsTWE?I zmwT2)Z92>mb|l*7{~Y{!&F>kctE;O*w5!_HSh3_iJS_7uvg)Udk&_m$cTs{cPY-C! zvImn-K^0E0VsJWnT!Gixd_k5@Ei0p_E^G=P9Q{pLY{>#LUhwdIA5w(9nPJ@~ES&V0 zF?meH0Wx#Dc)vf`VXL0+YcCj>8Y!k-W4M;~sr#22z~@FTHL=@HtTcU@F?y^oEzKKQ zDobgbY7%GiuHI?R4t=YWTRQG=wkc|dC-U*VO%`o4E_DUIwZ3nt^3bq6$xZ|2BaYJ% z--IxnU=mKtXyfjMM{%1H7LU`q6tE4apd48 9?BGVLW?TMejD6R7 z0IOU!tn>@7Q-F9L&QwBCRTz*#s=>fc(rbGd$I}Tjrbfh!8PE|n#Qx~&|9q4(;Q$Ty zs(BoM+VRI#z3+a`d7kV%`JE$tt;u(GF*Ji3WqZGV>8Il83*M^ihX(mztSvb-<#|ns z-oIDbB~{6D$oZ}H_l-yV8Of0}fVNU8SanLzA8sZS2!Fe#c|RyDLf6`8%+jd14BIif zW0$)Y36Mc8e5i<14q_AjXul^8Sn5+0wP*mc$t8KuyMZ>sthO0maW`U3x*DotN+Vmx z;&x@C5F?IMnqI|%tVW(K-Y88;`DufGii12prrnRTH1E6={or|UF`ZTTY}q5RVgK&E zUnmdrN+-d`SAO)F@;a}I=aqvTu0`VMSk2U$wMT(KAf;H#^HVq_sO@rKerC|PmPz|} z7Xnbgf>~>!Vt?0JDf8Dx?zLRxMoz0RU=R~!aCB4qIwql=2Nmuu|LO#u6I)E-v77g@ z1Ou0tBlvvXf+6w|K&g99(DbYsf9}#qOeQ-Uc99`JXQRvTtP02TkRoX7##`>M?g$YO zBa=zh0yB_vUEAeQHJ BerS~E?kfa`HQ=3cR?3+XHl;Dlt7FmFB== vu zm020cD5zvstSJhvA)J@?1E&u>ei&3lB (? zE`Rl!pi =roXmo{P_&BXy+5C?xhc0w^GU!K{O8*TQD XosJ!h`F+aAGWrLaMoy^gyF@pyv$70OV(SfgRseb{K zy=8&-%F6Xz?O-cOg&&x?C-w&J+JFbv>$?|@#(N+7ZGGxbPNv$UN}Q>_7OMaIaYd+p z-;Jf=xPC}_3Yb#JLKb);j5O$Tcx;XiJ}{f(@V5QxzVwIV>x{p&HoXAF!vsf8&}(ou z7PCiwEu_<<1ff&jnlcQk98qYWEMAmoE8G0z7J;C6fUFzc6xZXG#jclGtQ!qtxTV?6 zj;n1RbUfB}V%e<1fqs7=g9Y6(RXk0-w$3O1NX0jM`$iGJTcHPX_hR@XUp-wGT59w{ z(-pl5gO&7Mh90D+fhqz!$yoQEcWkroV3ez_p@)Gr^VX!Rz5V`TsyEKj|6*LY*}?x| z^Lb*eeegq~`L)+xc7*SjFPdL~n!t&R3lWd-ZT7Mqc8P?O I_iMLB?10 z1>wBYmc5%eAIH9!(LWUx!xcX(4$tjt8CbZ%U9@4uc!p15y qYVCjiU9 z{sSkVZccJU39WQ5KDRfe68XhDXD~UM9$3b)4_~n133lK^KwLFnis_90~}co1oEX zMq6MT>u^@%C*ww9Np6b)_Z#$a6>0xr$Zgv;*3CkboaB{|KCR*a06tNPpH-B(q%6G) zz&z5CsM)|@$ZY#u$`;n!XEbA0?wIY( oQXvtwIq0RzyImf zXv@9+O13R!Cx>a>icG}k`XH&ana}YZ*b|~L=mrj!NIkuAP2w{I%HGmUDz#?Kcdt;b z+Q>|Kmx64TQ$;|+LRJi_MXE o!@0-d+mc$h8MVXgqo4SpibD%*uN#R6tX@r8zwA-7Whjq82gDm$xxSzV zfZ wzNboI=e{-`x=3`INH#g&q%biIH7#b1$VkjY1l7^ zNVc`IQ4?YgW3bTzB5Mrp8MLDo86dbO^i9nzi#RvQ;9;WLER>ayT55c8$+EWN{_U~! z2Af;cbvl6iNG RmA%!6; z<5~=zsanEbi?%H6iamsPU(e+TYlkOB-(_e!9On?h+zYQdk1H!1ue)wNyR)RP*P@H8 z9^^F=jM?QL?bwbHM6?`8|3lgo$<9^lA8Gg8pZ9}!x^V22{VFC$f6R00r<15P%c_dB z*ZJ&8g?+G%*QmqZ6at#L*uC*I`dtGQBqQf_&;3~y z3PpzRr^lnwHc$PtTt6R*EWD|O9~S^Hi<3k`1V}qGcZ^>&zSsw1>7CUT8Ob;`U;Mf2 zf?DlG&{E6bCny~uOH+$6f55?hDIPr!PWg>q6 zRO zb+6C%*M^{)b*%n1^-HK<5mvu(nRD$CEWbdbmb403rE%T6y1TnuMx}&iRxUW$^TxIx zXuY9{uf0ellPjX1t^8Z)6)1l(B#rcnI3@L)FyWl%LhI-X753-xszY`cnDhWr&-pC# z+~_9p;B2EhHp1@>^?PSm?f{-i{eH(2T6-SHJyM}xA1k2hkqk;d# nF1|M=!0RQsxYt+LiDuh``S~h?^pU=w z+3$XUYAFgQ%4H-ZY4TKTW%MPaow4kQH|_cjk#Nl#r0`jh8E EgGEd`xU z)PT?1aZ+?hl2mnGEn2bG&gd>gcG3NrvU`^V`YtJ>M&nxgC*|tLdVhNgoTOkFnU68` zyC^a4XY% epS!oR!RrUEV|J;t|Lv&)#TdO0HV)iXk%tHf8_bOk{l&@5}z 5?L()t6=SE+fsdz5W+F8eaZk zMQZ56C)xCQxw4P>R SEM~mdR&YD>=P9%z@q!4(@qnF{~xu^qvkz8L~O_ zy(oqs^hq69KCfi=ETx`(zz?G`Bt+JGI u6jJB*a_F`#M1ajTRd2OQSiTIcG^Zh_cE34aSoo*w zaf4HqD+YVYLNOVhYzR(UpZ-gGOG(5aG!PCGc6RqR%8x~`O0S84Jw7KI4X#dt$j!$- z&d;r9kq}?dat!sBW$Iu2|L43)<`D3@NDr}18C^qdGq_&vbG5phTVN&Df;Tnu@RwQ} z8>o-iRn$4JwM*Xb-JJT6<)0JIvoV%)8WCsHg>_Z`K9d5y!t}(W?CtkF|5)Wnr-UoE zcGEMA=YxkKE`^tUN?#wJHMBKbUD9v&Pxeo4wof*g 1gT!dCKvyT6k9K gvZ9qb6={AOl7LM&iB`JbxaI%x)cdY7S w1q9A5+YSf-emgj&F7vOqj6(f!?Gn9fe7pfwi^ zk>qj*Gh&psd=F(4Tv_fP+Qh}dG#|fpD7 -n{ftuTCgJ11wRDj!u!=R*=gOgabvR+#w1;|9V{<8jo8vfPzEx& 9Fd^7^_`n3NCAVrT>snz%H+jJWbLy) jSA=u%u)MFfxmY4ll%B zi9VgmHP4d{s$9*_Fs&^vk#56_`bx{L#G&55A>rtKOSj+NRTHSI9q43az;&!VhBuFv zgXXwyqvm7ToaO7;(=NA4`mvrmF1@*(gQhUYM|jUR?%&dhtV8>^k?8Fx<|l!(pS2 aB7fAVsELef`QoZl1XE^;?0#H&d7 zWPYSPd4MnJ(pq2nt2G$>MRhNa;H((_^HYcgK)F(;b`Kx-9t9;r7Mq>#VS1>s{e8-? z#+nr>+yC1OkezuVT LS}!lGZ|=~J<9G2&y|;na|( z?@#2 oFtPnAFOf1(k;%=ihnQX-rnUjQ4`PnjP;re8L fBc4kpx4eNX|H$FY7# ee`a70Q3q-fPI#@=7w!i3~%GS*R zGbB1HV$HpI^2hG!-#+Mwi`U_F;4G0DFn{(sP9q!VZU}~a5={H>$z%Pg_h1;!<9El) z d$>|No_=FAz=z6jf7bo!=K~<*F#LF2guluFd=hj8a_{e!?CXISfiRyocuA zq!_rwL>6Yh{7?UClN@k7F(xY6y`mT{Y-g|Hx%c-sUX{QEV)=p2- vU@$Auc9i1hu6X&Q3E>Zy6@?=Z9aCCFw{#7r(gX)%i+-q5eEYA{Q zu0#gK;DSe2A0NmiT___*x x6KA_*lHHvex=&{8D4R zqJg@i)5ZZEKn33soIyh33)7*xo0vN>IrYQ?1^uvJ5Fa9cgK#U+!u@fQrZe@aCe_u93}TvU-C50k6nn36j=8@>X7 zm62YmyoRhUx7Fgv#Iyt^yonr=g(hOe#(o0OHVlny(D>KlM~+~85a61J(Wh;&08AbY zApqTgNdwAn*$MB>Q5ZY*GJkyf&%?u)(dK)5qQ`yisljjb*+HDpqJ}1+l;0 e9tFjPQ4Akv zY@* ;t zar*VSjM91W>F;`jx@6rp&v NZ**swzGQ21CjqUNNIm>w;L4W=*~SEPR}izNL?{0G?^qz1+a@=vgXx*JU+ zfr)z75!oVR*!GPPtiL{vqT91}1#+F9fgP~g?H%zWBC$AvhK?@JTw!Bp(d85NNSNCG zwo@w?snHjnUD-(fDBqb+9myWi*k@vQ-p~m-M0WLM9I1s zhG^`Bie|_-c7PE9L)bzpW(bp&3b|z7cMPrVmw4#19=_Pe3nYtbH9E@;NT3<0;T$)` zLB(H%`_H-j7jNyyHw^UB3qvuMc5f|N(iE=U-2*{HRUZF#&!4SrU=Jv`@5yC$O%LZD9jcn9%J`&;r2+O|>jppWpDy?gE)+V%PyyJH_M7WQ zFNA96$+Ogfejzrs2~u3sa)la=2JK~-Td%%Mequ5&TXn?2H!8o6eP=to*B@ypG%xo9 zeb`e}RT=5y>Ad|}=Zo?_p}f!HNFT(prK5+4dr+fX7X7n4&_x3m>qzaBep~K|;Y= zbahBg;6Aj)FvV-F!Pin59+Wns* KtIw+7s6J(OT`q4`#;?`B5rmg_I%yLllO-v zKz5~boH4f;{W&|DT96-){sWMTvAxC1f3xt-&S(RT8DQMjSHIUtOO^@{Urz;Q+AI6b zR?r}X`t5*{e6`!35TH{z4-Zt?L R^Rxy19D*erRnBxZ~zr=>r(jpWrI;D+e^&9 z3SIK_LDG|itgKdoCc-O8x+FwP=Wv!nlW0CzS)ya$4mTLh+GOF_t-*h|ZiV <*kDv+`RK8HDK?BJMA0+VRCx}|EP8-fR z+>Hl0N&pBXoGe%(i3p4=2ztX_ktSg#!5b$mhm3-p?5KPH(%U+di-!kK)#`BBNguJH zkPXLV`(k1G=?dj;_Ow{i^Nj?doGt@HJB{5P2O23YG}3sn?Z{{`!q}(@Bwaf4xqd3P zkbx#)qzjo2kjxaRe^Lifq aW_>Ug2w#AR00{v{eJy^&xf8C25#t0^!cU!O zHh(8m|1=JDue}_g`j$31yHb4nzIcCDXH``P@a<>z@e~!X$S5Zp#|8d8vWF-^c5;+% z{5i*WTW5m`ZkBcw29`F0V-Y+a6ZbElzu@n*<^RucFgnS5)c!+cq+!5*jMSP_pZDvh zAJc^|oHD=AP5iaL_hhoi@f^4JR#}w*yjfX-c63#*D$3&SVYaO;W}3P@sRO_c*qZ(R zxak2y?pf)-LVXQw)ER@Ob*(w@lQcK^l$Eg|;4E{!tN%Xdb9c3}-s)v*>XHZQ3@#-F zTG#vOhT|NvE*m)tNF!{rMM!E3p1khdNyMQZdgRbbMGIM}Hy*go#5(WW3)>GW}6j z-&Bw^Gv>GrRDiOY&DHeKz&G3Y)j=yr@}q|<=j;ZJQQp#t)Y7jY+GRRU={^&npci6H zylO)Y{q)wZ2N{BhM%rt)7_qILYFRd %XF@&*+Z^#(t7t_?XS(s|{6TTH@=SjEL* zfzs&srNxVX3aRK2;vy&*#_~AyUFK=SH=-O-_z<8WcqK)p{JVKC+Sr3JEFz+2NS{#J zw}b;%wI1ZaODc0zd)Zf}s57uEk4t%c hKxzS={W&GtX@N*p}VWsoiRQZJ+$+>nGO
e4eNKVxGJ5Te8sI-t`%%?`(@{yfPxP=RKwwJT1_evq&k6>nr{cljk8u(xf zo9o6A`TR!Ad&XYa(>H7~c&Jo+VtA1!5^;&YYe~@27;Yep6}^rT&u)ZyG~Efto!yz+ zt9e)^_ab&CqxNF@JUaQ5&v~D0Jar1IgmX~9E@RJPG;5~PMf5OB!zt}0U~3llDVh|~ zox((z?I45mdHs?mL4~Go^fXr3;2Jsk>Lt1Wn2;;ZUl?3~tANr1gDRIX9LLqMbA1zT zi`}`eKg(qg2u$of$$qC#Y3$n=gk${n^;?N#<9n-teyi-;bKn+qU3!zFH*09sT2-&r z#EUuGSdH;s>!Vt8rU<$LKY Ro8k;I@pN>s1IE>U|W{3lgV zf_atd-TFpp5CcNQofTDABeohFV?5eH4%LkdLOB}kGuF*3$8i30;6@T7i$91XqiPm! zB3-*wCT O4lLE`5(EPqvjg(-SZPejAqAaXwzg^~h?3xiNscP%^d(Gfy z!ZG)Yu4Eo(`f1phfhK=DaBpL{|IC%s2bVAp{N?1hIx?*lW~$d7>PF7t(6*Qi8Rx~j z%=|c#Hn1XgUOLVt-B?s9RmmFkzrs|^{Z*0t3Dp5q%d!2)kyCKepVvk*av&3cX$D(q zIiLS6!!o>dvrP(|=G_`Z?WjGk0DkO(VoDpC(W#6s8wuP(f&OR027Z^@n*PVYxS~qg zoQf>cdgq&)`sX9drlw|w_2cXC7kG}MchY+Tq}lrOF)tUgaLA#crR})R(}9dQ&41~g zwwL-?rA(*ZPBHZf3jK$PH`tq#8kk+atg{TD6}>K9==d)bvxJ`d6rt;82Rxs_32RR@ znytK}$xfc#Q=O>f4 `igZTwlwY>E4zwF}C1|Y0xnz|&BChb|n5z*V% za~5*dHYg_-9zMtz{%8PPq>Pp=5}BxDeM68MX{n%!&rjl9G-cDElcP~u-!!kSyS})( z*y4VD1-gv6Y;0`wD4nt4+mhvpEKH|~jHmKAEzlrK9k#nKaE-aLE|!U?QP EEYmR)abVPP6w1D^_*%V%P>cW%+PH{8i5 zk|afa(IdtN$cGTqN;bO)#w!iVihq37>8*ep?2j%WK{#1*W`YxMM;HqT21G8pTZQUd zv`=aixtP|8SX!WiWfuAxq9`kl|Yr~G&n zqz >Y#mlW9FHfGBINkw@hJ8pt{LA22at}8rqB07cp3U>*mb<6 z(wD}RGJ+Y~><7B~7k-lG0q&QtHl~S?00^l=6hdl*78ky6Ws)71Vut*(zv#}nAy>rg zE9occs`1N*^={@b_?`M*pPZZ+`Co6}{8&H63J<0u-+})m3Qkt?TlUFWh5G$7@dT4d z&R9}-dpl?lUTKEeIXJ9$`B~b!`nbaXY+Wnia4D?1YQ=~AjRv0z){Xum9zq~Nyo+wc z^8I5SB)wPA7U|{4R*+YTQIH-nq84*X)PR6q{;99ccVV+b0_N4Tp<*s-C&DNvyEU0? zHY_H9$fi9yYLS(KI5vWDd3_ODyozNLPPO>eY+1vA Q(Vr{VM04Jl{u@$E3BPsr+@NnFOv3V*&)pJ^>436K<*xc6b5yK GcZ+h8a|R3<@z_O z(b53v2sB)whB@Nb#d{yI>J_%!=c{I>Rcm~EZwIabb!iQ)iq1@g^A+Wg2djqv6EPpj zlT>~7)C6YNTg=9JHyQ&YEpzsfZ9C*4ha{H{?@*oIw8$p44C{SazS6clr1@;X!TaOt z=d8xR>8h$g(pHVM*tRr|myJQZ+5CFIx)WrDPX;dkn_*Iazo+rPcD|&3e?M@OyE}w> zg$#VH;+2eU3}x&Xm%j?-XyT~xl*#y5G bsCDWPB3gmg+uK#_%AL>vJ($Upj zUtL{o^V@qGe-suLZiZJbXt|@YmG~VAF3w8?Tm-}fdj Q(+YDMrR@tfsp~f~4A8-`j`u(ko>G51=QN1dk$qX_PbitQYsjWBnwG^5 zO~hC{vbd#uRMuz490ewnsFbo`n`194MRrz(30Diwu#A`?L4Fe#TNrv2*XTgH2%HoX zW8HT`^Vi`#SU$7Hm_U|}K*9p@MGkQ^hz=aBi(_nth!~6;EaomL(@n |lg4QUS4!)O#c7cXNuO2KYj*dJ zS|*$95N72wb#~#dXB9S^?>vE!^;a@k>i>^-a_2xu+wT% W&e5?{9Y(_~Pakt(J8nrg6 zYAF*g!kC#=+aVi;K@QVd2&Eqh-~LV9jmn>$KNk>TJ0lQn#*H;GeLVwGriBlUQeJg0 zbrS#r_s2Aq#XJU70-{UkV&4w;&)76VSC`goJ~rIP+yry*iQPyR`L__5x(z(9ix4hS z4z;7+3d3NJP+0f*!!1;hB&qnI-jbv1F6P?*^7;P3@#|)1wWx~dQD$%7g<9_59DRsK zbaQ6IQV)~Djg@1)v4NM9qpK@Vtoq@ne{k1a=5}d 9$Mu)lC^I&W!0=|jw*{aiub*&JtnY_h C_m)zAt+{?UINkhJv!IYfH&Q1uQK@b{i1aH4- b` zK*aOzqUajFS7*fkq jw|IY$YESjGeNJ@0ZDG22tJ%4?3YnNfo!>wYiP;oj?QC@xslBrHY);0@b@BXFP zB28Ga1J*i4_UJ-^udv+*C;3QxJ$TSKnPzFrR=7UY^Vc{*>OO018`i7z^m?gX!J)DW z$yO6I72jnxUKZt;5MUt;XZSA|`<3pXIPV!e3ml|K=LQ^LXOiozi(ERhR>-nP0PK0^ zpSS$uAO1h4zB($Z_WOQ7zyVacl^D8PQc4;D>F#c%8 EG!4dEf7EasQabx(n`IbDrnf`<%Vc5fIP5&0kJ!4z&2}rp3C!dCwEL8M(=>sFti) zJbiUm+!2$OcBK@2l;n-$|D(+Y-(vGwJ$gv&N^j88$6qU(w2GgM_g{$lxIR>{D@-}2 z61a54kH8Vs;zNS3#!V4644B`h3&vy{CJR~wY^S$pbSUw6bWr~mZ0cA_!bP4ox)ND& zFwocb{CtWT?>6Ej+JO!z__EzH3fw3Y(MjeszOE0}D*s~Y-Dy{WC*w*VS%}eHneZ{y z@b1%Wn`#7)#!m}ouToyUN)OVP7!uOAOQUSQV#q^PMdiZVJ4Y<=WPe94(*Ndhq`yM) zyKIUKKbAEs3pJV#sl!viWr@b7hGHW8`KaDf~{S#eB)&vtvk zBA3lg%_3PUi?)2Lq|8!M4ZMo$yq`u2)b>j8KRLt!nRNN8V;Qo}JPQ%81!(|ZrkQLc zlV~vBE1n_x+EU|;P@hN z3O&FmFCpQ(#p1*4&%^I|b-YV $r7~!-SA=ijQpD*VvSi4Ds%5jl)s+E&k?_z2Oa<) z$mgbV*82I3ZC^8G$6cID(AoOcl9>yd0S-lTIwRSo`;gVWEdf$SNm|IpmY*3PbQWi$ z+IiH7gE(C5sGsz^S*vz&a>{O$`-cF> 2e o-lrcfk#up5PFlSI;WBp8&?kEB1-oSq= I7!e*7 zC7-?3 ~HQdyXZv(vQO~ZhIRpnXzy|yV6<}BUW!QCUg5R_tszUpk`2{B#|hjwZlY10-Z zYTvt`jxU)%flicxRsoEXBw%nW%PiMdhj^uXB8l>xwck@u*Sw2!E?!>gF?8bo;#;N) z_~5aD%O-0!ude==WBnnH1A0{hzP#~Jjzsqs8>L#Sq8X9kD_+Ia;H`|>Dvod7I*lru zY*!UmIVlWb6B8S3lkPuxz e!Ail`=~=j1^QMB`Vqi{!eW94yTBxo`R7};=GtC zKt^_-XLc8>F3OsWvV5nsyz7a8zaD*NV}KFqu=>A(WcQxVn@+2o8@D@ujqJ4HWq-fh zt%vWGIk#y%FU_(EBdi;)mXqw-C?2jJW*%6BLBS6)RyoWYXli_G1tzz5(C104!&BS? zJ<7C#L-ogQMM+C&BD1&zx+b&t6Fs^`-1@-T3;0SJh{EB3#M?6*itkdE8FV5JHj zOWn9Q(hlO`$zR(KXsqwzN-^UAJoy Dfml20Jp1!*a^#ae>@Ax8 MK0vUI<+G+j*f zykqXw&P5#`-nbKW8G+NJa0Kc5yKRHru {iJ4f7NZF8VnuOG*D6S Y~*P&pD#cOx!AskJofhXB7etg&j&;SqL5Kbfo$HAKoZ@) v>Jy$(TE+I6oapkh8VrQcsg^m+#F0N-5GEAhozXXB4KL^f_J{gUOQ$Wh! z3VSDWXZPab)zXW6voV#n7xcx`(ZBW&(mjfy&ZnUU)S383Y bA!7dsLcswkU$4dPF{MW^u=}CQYp!7YF&mg#emyy zOsjyL)j|L<%TY@lMbr(?MwRBq%_Cc=XFV~dv1=Uoe~M$|cbdpn^^SOhtyG4SIYz#| zekPaf@%WGs!zI$}!@B eW)@2Bee(DOqp*s%YXZ4qLG g!AbRqUxQhHpvW4)+!_55Q8ue`4D znL8z)*JXC#${y7ntN58*vTn;?%EQU{PbEaARoC!?t=(hoI#oR>e=kD;Y%yyEGE7V$ z(~>Y-Q-tC|LAb+fQKzs%*Ke=zQL*2WZs>8Zg-J>PYFDR{lICd{5`1bMUo(ZN*I-kB za8(W||BXQ_TsbXL9J{JK=<0&@1V0d6LQu87z(3&Z78dy!42CQzWEb`fpttD1Ch7ke z9)<@3p?>*{&Mp`GB`&H4H(^9hhE)U<0>A DY%yBO@QHjqKyr< K=}g6@vv3W7p{`7UJd2t!XYhkjEkoM)T8H++2&EZAJmSox8BrM^v<^$1Lx!#$*}n zN^7x>CmVBz0 ~;!Bro1>owkFr-`b{ ~<@nBfP%jXE8ri8w#@ zuo2_l`hKm4)dfk?l`{PJkgLS^Ki5d!o4OW;<_{+W!)`-U`>R?x;3;H1kE6|7r$vP$ zr`-OCUq4T&dYhcRrZ8$TB}dP%NGf?M)oR8wZak(C*<^#G>V{(}LZiIJ7}?i?t;EFG z1b#_x$(+zwAw|q4sfP8xyvv5@(boNYk7;*Gw(diXj{p&aH441h;NQ!8ca=HD>+jg` zenDKO9fG%SWsnvIv&}WU-n-9(@O@X?WG%;m8gaKmr^c5}&TwNe` 9L(!z=YyvY7A0DD2nW~JF`pWSu0(RH61P5>#&VY2UauY20E0L0bKhZ zcs;Sv2{TLgcy%H9Lz#5>y}hBKBw&a!=NK;@akB_DQI-UnG-@9c!xp)|o_40L0zjVn z0~^{vjWiP|Lgp>-hh7Skth~ez=E %XrM2-CUXyGaA&tk8wFK=8C) zOup=*BQ;Qiu$5cRtG0RMiZC_=^~p9*3fwXV7X+)BO+ypm`iYU5d|c_w(yvey9_YNU z^b*z9!N$zY)UrXZiL^@hut TeK z5cKcG8(1Hxxq#7M;F#zKC!Rs?gaB)Ex(?FPxkq7=H`plc!=Ld}$2r0bq{Gwza}m-v zmE-G-_5*P$U}ZvD^|$43H&lb`>O_3;N=1X!YN8^toBDpF4Xj=IJF;^a`7F+!*Y=vZ zRcEaquWmGGE)9=A5!C|xU-DY93OVH9QH!OWSKXJZUS5)F1V(jv3<921k$^7ZsfB&B zXxM}QYFeRLUS^BwqHS~*WZ^UU)Yvs>%Vo)mqhQZxr{m;203ivs+ #NF R1la-k5Q59yZ-ES zKA9R`M G${9o$G *-PJ);c#?hVSFS&T &GZ~#pm?`yE@iC&_jT6J+GC%2hkfTF&O#mqajeEaYW(C-nMVd%vXqcv5? zk{Lsw0h7Jv!H(h*NlT8a;chj1*<&8NBKp{-3#-jyA1`xD4_P^he}Zx6T =$>>KG{6j(_FdX7MsjT1C#4((Z4lL@o$tFc-3)yhzR z*}mxp{Y0Jzp#UTHQ4^`?&i@%Tk)|3U!N~sF)y)AGcZPBs$IKKWf5H4O8`nzpdu>1F zIn)1%#Qy&8O7-0-JxB=5Du7v>6dX>HS6??wlHeXYa5e8`hi#W6=kN8X>gd`XDhXye zzjK4`Zn$elo?@`32(&mP%V;j3kF*|JlpA7}EDQZrg$?VixZHYOGTyI}QAYOsf|b^b z8qfXiIMlznd_XjKs|KBY9_s$}q+j{Rb;?8GMdv+1OkysR&BprrI!Vw@^YrHjz(~d8 zia`mqx=GgiCp@T|=8nsGRo}&*u}QK`WS-+og@pjcPeQ#p3X=YEH$#E{75G+vZxmZ+ z!|xBM<~p{M4Nb(ZFK$c=xrmwi6@Z~0jLL9K+x^2SM~}`db)5hyw%5kJj_MVZ&mD=R z35hF8u~i^XUo$;1JG1`wlScpCxekDeYKlP(I zSDLzXd>nZgymuM2)!cD~n4M-bvo>?JX3#Ap{?42jB`rNiEtOLl!(^^ug5qWotLsjZ z8Xg{pXA^*`%bMtvUfyDA(rdWcdp*$`g*%9pPP9V;Zzl8QTp(A_a-8KkSgXH-J9q{u z^>CuATcxLY?=1?KNFto 3Oy>MP30tb zK-$>6RNzO@r8r$QW{7!S6cG543%MnM^2~jD6^;w>K<3W}@$f)0mY=;HNGIuGSPO-= z{(VUFY?e?D%n+SW&fd~Xr?(S+&X6{g*8j2rZVz8*eK1 peO_JV34_1yI@|JF8;f3Au?Kj)@f=+g1wXp5tPULbyiTZkxC>}UO;?@38a zc1kVzgbo~81t*yY@71YTYX)992|iAl-iOsKCdGc{sIhvNmf&yaS-Eeky0%I#x|QBZ zOxyH%+0OT3N4IVW;?{{Lio6Y;!RW}n%>arp#cTsc%Q`WytEZZwsXDakL3iuBBleybh zYLk4_b>N!V4Y)4FXglU=ct1o3s+DYIK$)c3J?E8o)vDX>kui_50tZLUk@fH8pv%3^ z%g@1UcbA<>TS7kXr!IfC) ;`C X}T9r+ThO3%Lf;a?~H@F$QkU?DQNfmeWj&A(xB*7rcWU1I#~{UeA%dw zFy|S(fm$tZqk_5vd)!A)@#d;u_X Yz+5E{|CQx+c>rZ%jEiAYd zQ4XEi%3l^SO(GnX@J1YZLjnaoY4$+)LQ2TeE8;EG( ud%ur5n-q1=&%LCRmoaF8%Yy8=iqvzwa!L=|ufW$Z#q2gMUjBZyZeL*vEMiB)K zACP$>BR}Zz+GB&gEDRJ0??kG?(#q8Wk?$%I0DyF5T%~m}(m~Dk1c72M6{XyfO@ELI z_y@!%X2kwuY*BLtv;w&vsmlz$xS7TaqGNUblsio`DR-=R>} M$_mml;yAgmf7 zJ^^(-5ETn+zDP6E9?a;Ty>(?Oczd)n?cH%Q9{RW%31}#C?$0g{_ZJuD+GlJVz5rK& zAs 3{?^aQV}5pyB{VOZ^}=oO*{cdhmj8HyiVBvSmm^}Hro zb|au3TUw{ew7nh=*vCe;{(agH>mX=SQ(hqu$!nRRktys-kt8EkAGv^B6vvq0s>+^$ z-D*c^G`FovVaOcDWwaTwHHHlq3GKlne;MgMQeMSBvA4W;dCdr4%3VIhk;~^BnX8E+ zaHup_RFjUvQ+qD3?~sv=E%8;uqmuGdcs$FK?z-Ay&<7w*w3d5irUBX!C@LxoM_S;@ zc5pcYxL{1yE5ysg8#6Xf)T_ >vfXs=Q( zW?<`EtgZGW#x3sPjaa4&=^kUPC0pd _H?FT3dH~ajE-9 zc=4Z=;-3#fRR2Qfmwv(Lav{Z{&h-4%=l~IJ>S5}PPU7~N*oRhe|6S+i`yLbFE6$+Z zv(*m~;h+77CT;H-n1|JEUuL*;_}yGBtp!;`Hc=knoJx*6aPym^hA>DKOJ)H@p2lLL z{1Wtk7as(sRg0Y8dsm#~380=B<8pFd^mR%-rB0Ewo@$k pK@epavSd{mYkk$7xCb zaZBghk-zIxkL-`W#1GkW#iI7vTxvS2sF#vuk7mLzbYYFgrU1^8j}kQY&}3cz?I*aT zmfAlMmO?}t2#<;J!L?*7( Op9q7Pk%U%u+f953V-{o{hG#A< zFRvzzm*)_t>m84rae##fK *MCm)5b`gW|w8B>Pmrc@8t~>pDiI|Dcbed6h*=*1|LN`+1fjmxtkRV zn@vI2<6KOSs`w*)3yeWFHP)rd2G$lg8>?=5`^Og&8JZ4o!{$gc6kKl99$CliYOW}* zJP{p7kJ=X=){ mqDCNX9$=ov{h%?hDQ&>C-DA`=ZpWOR##|b?8jzK%{F%Bmh`cr( z13-!} t3% zKWbfDUHyL%oVYjCj}EZ^lKXFgfB3_{jvg|kXEnNgJ7)(s)^+dj{(7?W8g`YlhaXni zdHQ)1mLPUhTzfaoGrHxX`a6+!_x1U3A%V;!a{gzjarLEY$MyZt5#X7hb)qB~ub&&h zNB(n%uNi8v@Q(Ep5I`w5?{|LlpqoDl#3nntxlM98PoU+Mk1UmrixYIN>5|Ty+@Y2V z!{yevY)R|R8b?k67!#qoL5z(V)Z^@Xflcn`A1k<_0qn-FsJqpyWtmYWid(-@A~(8n zs^FV78|N>7--k8i@1A|R(Osh;m(QjIBc+R8LyC!`DY(bX#90I|BqcJMg=wm@uVJh} zoW{)p&1IntksoE8EZl9eAp|o91o14xE*lO%?Z3UvJ>|qDB@HoLFMcZiv_K48uOp!} zG5&Zrxx2lXypuo|4 ze1l^QTUv5%5_v~EAb8;kjQSk~Q^#uDbrmWIzcJL!39r)6qI-9$zVAT)7K1e7-qrA~ z!?E6_Qp`bF2Y}+xizU)XWVl{=>5bku{ 4z zUOQ^qj?aSXh~yPd0re;$Q*vYOp*^TC?V!5KI{SzoHdM!_CIZCTnqJlwxiep2n;;wC z^o;>hgE!noQM#16(wo9cDS1q8k_}-%2`DvAk#qnWJE3-63zV9=jnI-1RQ?SMpszR2 z!O@Vvp~{c T26vCs%ik5(spOB?Elv46 zQRPDbz39A3$<={i23@!Z-|y?(U!2bqqUF3y7+N0#KTeQS)uJ4;;!YTn9|Xuwje0Z{ zHf0pp;fD-aCQn^WoAG3IT+PQZsy=Hzf1`>+#ox^6-Qil-kn+V-i_1at)8~s%op;yJ z?jZ6P`}^b8( yj9zUUKzy7 AmwaIhm|Fhhgw}zKbk)pvF$V87(aLve zK;sR2o6EM{?0$E4t4aA%g$C!PF$J5Q jfAjEs>+E$!99fO6Z3k15EOrH!wOQ z1|iOA&6X85yT9Q0EP3DXG;jE|yf-UNJ(C+pIn1GTxDSyPxeu8TjfJwWbXh~2D$~*^ zAR_#lg$Bq1c#4u9j)GE&J516(f_G^566 C+=n8 z4!Lh4ec{F)_T`6(x< z#rW@+&%-MI#z;5+_s)wfhXd9A^{pVhuHQKKbE}7Uy=HGm3P;l?bFN2ps?E>j9!7Vn zZx0--8_~LppZdDFxdjB+S_39F-MyFEJ>4Z+XSV;<0yw-x{}PXy`etBZ!GJ{B2mq?< z>`YO2f9QS 6ob#U*aZ$9vJOK%U@L;2E-G=@M;g zeREFLtxxH%OMZ` ;m*lJrh<_Sbeh{&s$x;Up?hnOfdS|+n{thKfi@0 zK2l_2a@0~RxnM=4VBqFW)c^gB-*^u}O@dV9N`ASwWF@6Fzx8Z^hMIDz9iNb`bsidx zW=LMTrqXp4gIZFev EzQamQ_unORYnlxD@|Y!DfsN@c27NmElLjkVqE?KM z_)i?jPBF)!o0>5&ku?%fA^w+V-%^~b%nkR-{X%R6Mi6KLR#=Zb1cWOGN~cCJ+}#g< zuGY 68twSo=6j`dcI;O=BGIg-TgvW>D4guHMj2@U4VlwJ>wLK)o8Sy*0b z@hkQA!?^3)m@-x_YBgZ;xdH!anS7`UD_IX Z_GzDEw*^fwt$%* znb?-1X36DxSDzPCMR9(j<(|?Hu74e))$V*<&3ey~x2HqZIyz~@x^b%|4{dg_w|_xh zT~?Km@9D5`QK$cb8u-8K*G;O)!@0>rRN28mZ*}?2e^HG_f~hC)_pUu^ihL0ctHbI$ zb6@T|)`#=e!-w_VXB8tZz^uDgac>AQ{5lVz;?vRsRpWGSN% e{u?_hWql7aSnie?Jm^!U?1!E z=N)e@L=5MhdBp-;`N;WQAdTOa5V&H#UQ@e-UG4pljf?zDKm!bCVS~^Ph8R7hLhj=t z-VB`6MSow+7=2~KSk|Yt`&39o>>hDLm0e_%LAn0svHsw6VTPmlE8(Eu>&-#y ^UtgN zS6Y_N_M3c`pO+H9!*k(GIh7!O=emb^Y-^wv5GAmYIJDlb0ltZW14tB`Z|-jZqR^ zyrvdSzx#f{ZUKmpKmimV_7*9%>CVbCs;l-G%6Z;0F_S<9l(5Hd^O!Q5bnvZ8DA?M! zW*q5XJ=%ta-cSYo?iLZh&KLK(K{$x``zH+1*~J&EDw>^>f*aJe6&k4U(G5BDKq+zT z4#k2s{*gDy4A_bs4gz*FKiQ3Kha3xMO%2FjV6bd+6ojf9!d7eaj~Q-uGZV=Lk&~^$ z@YD~~b)6(<2S6W*Uqj72>YtE{yl+tz^Kx(drgW8*{!)qY#pbh1`>S| |2YQneG?yY=-fcg_rIqU6|i&Uyu{pX*V?0vlYhEJ^_X)3YL}=!e+kDVsdEL z7vu@`CMYSCSL$Hxv^(uM-saKz58+f@^TXZvqatneglHPlVh6{3GN 7i+X_2BZ1w=Yxc%h?^w* FQ@i#VvqSjI}GiB4AFf zH5;JLlDhvwoceeB+uMb5VDtZu@<03b(J@|~^WXhYrzoqps=iYqcy;2UlVAMe(N)gv zIQBCI(ucI1PG@j>b^F47zKe0WndFh(3gQrQ*?lS25}bH(3(ZJ}453lSu{EbB$f7hv zLpRR4r-N@*JDf+*Ltq{}8Ok>GFer5Y?98!N*LAD$xN??cK>*g$2sPeYd@cbZP*z+7 zrw!UIeZNgx6G1lMzkSl>!2^TC{BF;VCPO1PpZ(f*?9knW3v@I*Y?1)jV58o$V$7Gm zJnT |I}|fA((DB-nc6s1l{###soWTG&e%ql=G}F?_H)sC`7zBQ 0Vpb~K$n_XCw?8=#)lO)2pd}M7YF%(lC^HP{3u10}{XEC7GvI^A(<@bf9Nh2^ zxqjanc9cxF-nL~*Mv?>_*2T}Yik`ZWFeIFSDt(n!_6YCYMs2fd0TYo;qwl&h*Rg`X zEYHuXyuwI}Lpa0Wl~Ds+0^Z&N?)lyDrx-jo#Tgrpu;JU#DR$Ex48v_%lBnL&XDqhk z_|*#^>*!(HM$zxsvaVzJPuu)zRTL!N`N{O5yz_bVEmYxq<8 SmxB#JNpXY9b*q|Z-2^%rN#HZ>%(CQSV#$voJr9c>;ZQ+lU;%(%Fqlvm!lCjJ!J zEM3NPFFHD~diUn2BlQ6K&DYNnD!iNRnav~RB$WSQ7PMtmL*lBN0IN%(bmzYRgDvg& zQqZS=PJv%#N82ms&*UAv-JBi ztV+Otj#iB}aus@`s$%W(cKUeg+9R* !^B@6HDg__rjtbI?sgO>nY_F-M z6`%M-jc~SxDHFATF63qDTpQv0E+t174xt97muwMZoS}K@Y$i}U#$438xtM423TTiY zXU5b<4I{VvFqeSSj7v( zFC&f+E6Kshu(~#bf*B*fqTQ~?-@W;4nwkYok->4&CWhaEU%NT#n{iWDIp453R!WJX z#m7c=jR}@oorIPJB1GaYqHWp%pCxm1-^$RVwsF7Crk-M~eZ-zv6Ow^gN*loSyr|=Z z+aauBqwLMj$qFrLY2XY4UpwQDrd*LY=T=Jf-740`IT#a&Ytcx${M)GcN%m!-7IR)z zUZA)3UY>aWk1X?_c@ZKqlXsZM&V^c@uyP3_>t4w332o2^rROI{0bk>JveK;!=H=DD z#!JGh6?QUS)s#uT=;Mm1YSWIjnw@t&6rrJ?*B|Mt7+h_WKiJy+Kr=H*53K^-mCiR> zpU!s#P@vF`rH)meH#PoN68aavQ2!QnPpgjEv)PLHhp7GC-OGUg!!G4L%Xh&!DXZ_^ zA5YwVb2spMnCGOti|c&1gMyjRc{Xd3XEvt#;Cp+r+}7YkQfE{Z&rg6xMF87hSxJMm zPH$$Wl#4iL>8`Js(@zP)+YJg6WKVRr6ux+T%N`UR>Co-qfoF!*(%EQac)UzY#R@^+ zaHJWmn?a}ykENu-n0OyWru=jc6B;b79qJac5dZts4tT~0tFRP`01wJn?GM{4z5V93 zh8#%#g&bn?hlX8TpfrqZOu_=ZR)jQE1Z#Dzo3G{5du>!deMbH~Q&stdyyx$Ym&mq4 z;wv3!R8;-etU314zS^;ETdv(d52D*FZ>*l2&7TkrXnm# WPPpB(rd@EDr3i9+DgCMtV1ZG+N^<% z$$%%NJGMB!WSGH fxh`&RIH_?~Q=!P8HL1gnrZ2WdN$Cm~Wac_FoEn|^v}AWzgC zV?MZ}kPYt$LdeElz`HqfY|$>+-NBl^pvm$|cnmUDnRdy-(SAK*`3f#AaS$IdCWDdZ z5UkJt2Fq->&Uu8g+^l-*S(ZFgoL5QI>oECebY69?Btv^_WP`@s`eT5i6sEN@(iCmY zU~LV1b$jw8w9=Ty?A$w+?%m9ksgfYr)UGv3E=c#>&0X3h3;1NXv=&x(!H#>wZK^go z!eRn>HTRTa%6pd6A37+SKo@8Gs@YlyDLwgbTMwx6{Z#E=DP^o+@gJH;{g;T&=-u~r z`ajjv;lA+>zPfG7`CTv4X*&3D72K+p^5X92=Ml>-EB%+*G?*td