feat: support download of huggingface files from a mirror website (#2637)
* fix: load image number from preset (#2611)
* fix: add default_image_number to preset handling
* fix: use minimum image number of preset and config to prevent UI overflow
* fix: use correct base dimensions for outpaint mask padding (#2612)
* fix: add Civitai compatibility for LoRAs in a1111 metadata scheme by switching schema (#2615)
* feat: update sha256 generation functions
29be1da7cf/modules/hashes.py
* feat: add compatibility for LoRAs in a1111 metadata scheme
* feat: add backwards compatibility
* refactor: extract remove_special_loras
* fix: correctly apply LoRA weight for legacy schema
* docs: bump version number to 2.3.1, add changelog (#2616)
* feat:support download huggingface files from a mirror site
---------
Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com>
This commit is contained in:
delta_lt_0
GitHub
parent
978267f461
commit
5ada070d88
|
|
@ -54,6 +54,7 @@ Docker specified environments are there. They are used by 'entrypoint.sh'
|
|||
|CMDARGS|Arguments for [entry_with_update.py](entry_with_update.py) which is called by [entrypoint.sh](entrypoint.sh)|
|
||||
|config_path|'config.txt' location|
|
||||
|config_example_path|'config_modification_tutorial.txt' location|
|
||||
|HF_MIRROR| huggingface mirror site domain|
|
||||
|
||||
You can also use the same json key names and values explained in the 'config_modification_tutorial.txt' as the environments.
|
||||
See examples in the [docker-compose.yml](docker-compose.yml)
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
version = '2.3.0'
|
||||
version = '2.3.1'
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ if args.gpu_device_id is not None:
|
|||
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_device_id)
|
||||
print("Set device to:", args.gpu_device_id)
|
||||
|
||||
if args.hf_mirror is not None :
|
||||
os.environ['HF_MIRROR'] = str(args.hf_mirror)
|
||||
print("Set hf_mirror to:", args.hf_mirror)
|
||||
|
||||
from modules import config
|
||||
|
||||
os.environ['GRADIO_TEMP_DIR'] = config.temp_path
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ parser.add_argument("--listen", type=str, default="127.0.0.1", metavar="IP", nar
|
|||
parser.add_argument("--port", type=int, default=8188)
|
||||
parser.add_argument("--disable-header-check", type=str, default=None, metavar="ORIGIN", nargs="?", const="*")
|
||||
parser.add_argument("--web-upload-size", type=float, default=100)
|
||||
parser.add_argument("--hf-mirror", type=str, default=None)
|
||||
|
||||
parser.add_argument("--external-working-path", type=str, default=None, metavar="PATH", nargs='+', action='append')
|
||||
parser.add_argument("--output-path", type=str, default=None)
|
||||
|
|
|
|||
|
|
@ -614,12 +614,12 @@ def worker():
|
|||
|
||||
H, W, C = inpaint_image.shape
|
||||
if 'left' in outpaint_selections:
|
||||
inpaint_image = np.pad(inpaint_image, [[0, 0], [int(H * 0.3), 0], [0, 0]], mode='edge')
|
||||
inpaint_mask = np.pad(inpaint_mask, [[0, 0], [int(H * 0.3), 0]], mode='constant',
|
||||
inpaint_image = np.pad(inpaint_image, [[0, 0], [int(W * 0.3), 0], [0, 0]], mode='edge')
|
||||
inpaint_mask = np.pad(inpaint_mask, [[0, 0], [int(W * 0.3), 0]], mode='constant',
|
||||
constant_values=255)
|
||||
if 'right' in outpaint_selections:
|
||||
inpaint_image = np.pad(inpaint_image, [[0, 0], [0, int(H * 0.3)], [0, 0]], mode='edge')
|
||||
inpaint_mask = np.pad(inpaint_mask, [[0, 0], [0, int(H * 0.3)]], mode='constant',
|
||||
inpaint_image = np.pad(inpaint_image, [[0, 0], [0, int(W * 0.3)], [0, 0]], mode='edge')
|
||||
inpaint_mask = np.pad(inpaint_mask, [[0, 0], [0, int(W * 0.3)]], mode='constant',
|
||||
constant_values=255)
|
||||
|
||||
inpaint_image = np.ascontiguousarray(inpaint_image.copy())
|
||||
|
|
|
|||
|
|
@ -485,6 +485,7 @@ possible_preset_keys = {
|
|||
"default_scheduler": "scheduler",
|
||||
"default_overwrite_step": "steps",
|
||||
"default_performance": "performance",
|
||||
"default_image_number": "image_number",
|
||||
"default_prompt": "prompt",
|
||||
"default_prompt_negative": "negative_prompt",
|
||||
"default_styles": "styles",
|
||||
|
|
@ -538,6 +539,7 @@ wildcard_filenames = []
|
|||
|
||||
sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors'
|
||||
sdxl_lightning_lora = 'sdxl_lightning_4step_lora.safetensors'
|
||||
loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora]
|
||||
|
||||
|
||||
def get_model_filenames(folder_paths, extensions=None, name_filter=None):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
|
@ -12,7 +11,7 @@ import modules.config
|
|||
import modules.sdxl_styles
|
||||
from modules.flags import MetadataScheme, Performance, Steps
|
||||
from modules.flags import SAMPLERS, CIVITAI_NO_KARRAS
|
||||
from modules.util import quote, unquote, extract_styles_from_prompt, is_json, get_file_from_folder_list, calculate_sha256
|
||||
from modules.util import quote, unquote, extract_styles_from_prompt, is_json, get_file_from_folder_list, sha256
|
||||
|
||||
re_param_code = r'\s*(\w[\w \-/]+):\s*("(?:\\.|[^\\"])+"|[^,]*)(?:,|$)'
|
||||
re_param = re.compile(re_param_code)
|
||||
|
|
@ -27,8 +26,9 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool):
|
|||
loaded_parameter_dict = json.loads(raw_metadata)
|
||||
assert isinstance(loaded_parameter_dict, dict)
|
||||
|
||||
results = [len(loaded_parameter_dict) > 0, 1]
|
||||
results = [len(loaded_parameter_dict) > 0]
|
||||
|
||||
get_image_number('image_number', 'Image Number', loaded_parameter_dict, results)
|
||||
get_str('prompt', 'Prompt', loaded_parameter_dict, results)
|
||||
get_str('negative_prompt', 'Negative Prompt', loaded_parameter_dict, results)
|
||||
get_list('styles', 'Styles', loaded_parameter_dict, results)
|
||||
|
|
@ -92,13 +92,25 @@ def get_float(key: str, fallback: str | None, source_dict: dict, results: list,
|
|||
results.append(gr.update())
|
||||
|
||||
|
||||
def get_image_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None):
|
||||
try:
|
||||
h = source_dict.get(key, source_dict.get(fallback, default))
|
||||
assert h is not None
|
||||
h = int(h)
|
||||
h = min(h, modules.config.default_max_image_number)
|
||||
results.append(h)
|
||||
except:
|
||||
results.append(1)
|
||||
|
||||
|
||||
def get_steps(key: str, fallback: str | None, source_dict: dict, results: list, default=None):
|
||||
try:
|
||||
h = source_dict.get(key, source_dict.get(fallback, default))
|
||||
assert h is not None
|
||||
h = int(h)
|
||||
# if not in steps or in steps and performance is not the same
|
||||
if h not in iter(Steps) or Steps(h).name.casefold() != source_dict.get('performance', '').replace(' ', '_').casefold():
|
||||
if h not in iter(Steps) or Steps(h).name.casefold() != source_dict.get('performance', '').replace(' ',
|
||||
'_').casefold():
|
||||
results.append(h)
|
||||
return
|
||||
results.append(-1)
|
||||
|
|
@ -192,7 +204,8 @@ def get_lora(key: str, fallback: str | None, source_dict: dict, results: list):
|
|||
def get_sha256(filepath):
|
||||
global hash_cache
|
||||
if filepath not in hash_cache:
|
||||
hash_cache[filepath] = calculate_sha256(filepath)
|
||||
# is_safetensors = os.path.splitext(filepath)[1].lower() == '.safetensors'
|
||||
hash_cache[filepath] = sha256(filepath)
|
||||
|
||||
return hash_cache[filepath]
|
||||
|
||||
|
|
@ -219,8 +232,9 @@ def parse_meta_from_preset(preset_content):
|
|||
height = height[:height.index(" ")]
|
||||
preset_prepared[meta_key] = (width, height)
|
||||
else:
|
||||
preset_prepared[meta_key] = items[settings_key] if settings_key in items and items[settings_key] is not None else getattr(modules.config, settings_key)
|
||||
|
||||
preset_prepared[meta_key] = items[settings_key] if settings_key in items and items[
|
||||
settings_key] is not None else getattr(modules.config, settings_key)
|
||||
|
||||
if settings_key == "default_styles" or settings_key == "default_aspect_ratio":
|
||||
preset_prepared[meta_key] = str(preset_prepared[meta_key])
|
||||
|
||||
|
|
@ -276,6 +290,12 @@ class MetadataParser(ABC):
|
|||
lora_hash = get_sha256(lora_path)
|
||||
self.loras.append((Path(lora_name).stem, lora_weight, lora_hash))
|
||||
|
||||
@staticmethod
|
||||
def remove_special_loras(lora_filenames):
|
||||
for lora_to_remove in modules.config.loras_metadata_remove:
|
||||
if lora_to_remove in lora_filenames:
|
||||
lora_filenames.remove(lora_to_remove)
|
||||
|
||||
|
||||
class A1111MetadataParser(MetadataParser):
|
||||
def get_scheme(self) -> MetadataScheme:
|
||||
|
|
@ -385,12 +405,19 @@ class A1111MetadataParser(MetadataParser):
|
|||
data[key] = filename
|
||||
break
|
||||
|
||||
if 'lora_hashes' in data and data['lora_hashes'] != '':
|
||||
lora_data = ''
|
||||
if 'lora_weights' in data and data['lora_weights'] != '':
|
||||
lora_data = data['lora_weights']
|
||||
elif 'lora_hashes' in data and data['lora_hashes'] != '' and data['lora_hashes'].split(', ')[0].count(':') == 2:
|
||||
lora_data = data['lora_hashes']
|
||||
|
||||
if lora_data != '':
|
||||
lora_filenames = modules.config.lora_filenames.copy()
|
||||
if modules.config.sdxl_lcm_lora in lora_filenames:
|
||||
lora_filenames.remove(modules.config.sdxl_lcm_lora)
|
||||
for li, lora in enumerate(data['lora_hashes'].split(', ')):
|
||||
lora_name, lora_hash, lora_weight = lora.split(': ')
|
||||
self.remove_special_loras(lora_filenames)
|
||||
for li, lora in enumerate(lora_data.split(', ')):
|
||||
lora_split = lora.split(': ')
|
||||
lora_name = lora_split[0]
|
||||
lora_weight = lora_split[2] if len(lora_split) == 3 else lora_split[1]
|
||||
for filename in lora_filenames:
|
||||
path = Path(filename)
|
||||
if lora_name == path.stem:
|
||||
|
|
@ -441,11 +468,15 @@ class A1111MetadataParser(MetadataParser):
|
|||
|
||||
if len(self.loras) > 0:
|
||||
lora_hashes = []
|
||||
lora_weights = []
|
||||
for index, (lora_name, lora_weight, lora_hash) in enumerate(self.loras):
|
||||
# workaround for Fooocus not knowing LoRA name in LoRA metadata
|
||||
lora_hashes.append(f'{lora_name}: {lora_hash}: {lora_weight}')
|
||||
lora_hashes.append(f'{lora_name}: {lora_hash}')
|
||||
lora_weights.append(f'{lora_name}: {lora_weight}')
|
||||
lora_hashes_string = ', '.join(lora_hashes)
|
||||
lora_weights_string = ', '.join(lora_weights)
|
||||
generation_params[self.fooocus_to_a1111['lora_hashes']] = lora_hashes_string
|
||||
generation_params[self.fooocus_to_a1111['lora_weights']] = lora_weights_string
|
||||
|
||||
generation_params[self.fooocus_to_a1111['version']] = data['version']
|
||||
|
||||
|
|
@ -468,9 +499,7 @@ class FooocusMetadataParser(MetadataParser):
|
|||
def parse_json(self, metadata: dict) -> dict:
|
||||
model_filenames = modules.config.model_filenames.copy()
|
||||
lora_filenames = modules.config.lora_filenames.copy()
|
||||
if modules.config.sdxl_lcm_lora in lora_filenames:
|
||||
lora_filenames.remove(modules.config.sdxl_lcm_lora)
|
||||
|
||||
self.remove_special_loras(lora_filenames)
|
||||
for key, value in metadata.items():
|
||||
if value in ['', 'None']:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ def load_file_from_url(
|
|||
|
||||
Returns the path to the downloaded file.
|
||||
"""
|
||||
domain = os.environ.get("HF_MIRROR", "https://huggingface.co").rstrip('/')
|
||||
url = str.replace(url, "https://huggingface.co", domain, 1)
|
||||
os.makedirs(model_dir, exist_ok=True)
|
||||
if not file_name:
|
||||
parts = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ import math
|
|||
import os
|
||||
import cv2
|
||||
import json
|
||||
import hashlib
|
||||
|
||||
from PIL import Image
|
||||
from hashlib import sha256
|
||||
|
||||
import modules.sdxl_styles
|
||||
|
||||
|
|
@ -182,16 +182,44 @@ def get_files_from_folder(folder_path, extensions=None, name_filter=None):
|
|||
return filenames
|
||||
|
||||
|
||||
def calculate_sha256(filename, length=HASH_SHA256_LENGTH) -> str:
|
||||
hash_sha256 = sha256()
|
||||
def sha256(filename, use_addnet_hash=False, length=HASH_SHA256_LENGTH):
|
||||
print(f"Calculating sha256 for {filename}: ", end='')
|
||||
if use_addnet_hash:
|
||||
with open(filename, "rb") as file:
|
||||
sha256_value = addnet_hash_safetensors(file)
|
||||
else:
|
||||
sha256_value = calculate_sha256(filename)
|
||||
print(f"{sha256_value}")
|
||||
|
||||
return sha256_value[:length] if length is not None else sha256_value
|
||||
|
||||
|
||||
def addnet_hash_safetensors(b):
|
||||
"""kohya-ss hash for safetensors from https://github.com/kohya-ss/sd-scripts/blob/main/library/train_util.py"""
|
||||
hash_sha256 = hashlib.sha256()
|
||||
blksize = 1024 * 1024
|
||||
|
||||
b.seek(0)
|
||||
header = b.read(8)
|
||||
n = int.from_bytes(header, "little")
|
||||
|
||||
offset = n + 8
|
||||
b.seek(offset)
|
||||
for chunk in iter(lambda: b.read(blksize), b""):
|
||||
hash_sha256.update(chunk)
|
||||
|
||||
return hash_sha256.hexdigest()
|
||||
|
||||
|
||||
def calculate_sha256(filename) -> str:
|
||||
hash_sha256 = hashlib.sha256()
|
||||
blksize = 1024 * 1024
|
||||
|
||||
with open(filename, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(blksize), b""):
|
||||
hash_sha256.update(chunk)
|
||||
|
||||
res = hash_sha256.hexdigest()
|
||||
return res[:length] if length else res
|
||||
return hash_sha256.hexdigest()
|
||||
|
||||
|
||||
def quote(text):
|
||||
|
|
|
|||
|
|
@ -368,6 +368,7 @@ A safer way is just to try "run_anime.bat" or "run_realistic.bat" - they should
|
|||
entry_with_update.py [-h] [--listen [IP]] [--port PORT]
|
||||
[--disable-header-check [ORIGIN]]
|
||||
[--web-upload-size WEB_UPLOAD_SIZE]
|
||||
[--hf-mirror HF_MIRROR]
|
||||
[--external-working-path PATH [PATH ...]]
|
||||
[--output-path OUTPUT_PATH] [--temp-path TEMP_PATH]
|
||||
[--cache-path CACHE_PATH] [--in-browser]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,10 @@
|
|||
# [2.3.1](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.1)
|
||||
|
||||
* Remove positive prompt from anime prefix to not reset prompt after switching presets
|
||||
* Fix image number being reset to 1 when switching preset, now doesn't reset anymore
|
||||
* Fix outpainting dimension calculation when extending left/right
|
||||
* Fix LoRA compatibility for LoRAs in a1111 metadata scheme
|
||||
|
||||
# [2.3.0](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.0)
|
||||
|
||||
* Add performance "lightning" (based on [SDXL-Lightning 4 step LoRA](https://huggingface.co/ByteDance/SDXL-Lightning/blob/main/sdxl_lightning_4step_lora.safetensors))
|
||||
|
|
|
|||
Loading…
Reference in New Issue