happyz
/
Fooocus
mirror of https://github.com/lllyasviel/Fooocus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add words to tokenizer 

thank you, GPT-4
This commit is contained in:
lllyasviel 2023-11-01 15:43:49 -07:00
parent 01bfa11fd1
commit 7d532b0285
3 changed files with 99 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fooocus_version.py
View File

@ -1 +1 @@
version = '2.1.772' version = '2.1.773'

528
models/prompt_expansion/fooocus_expansion/positive.txt
View File

File diff suppressed because it is too large Load Diff

9
modules/expansion.py
View File

@ -33,13 +33,20 @@ class FooocusExpansion:
positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'), positive_words = open(os.path.join(fooocus_expansion_path, 'positive.txt'),
encoding='utf-8').read().splitlines() encoding='utf-8').read().splitlines()
positive_words = ['Ġ' + x for x in positive_words if x != ''] positive_words = ['Ġ' + x.lower() for x in positive_words if x != '']
self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf self.logits_bias = torch.zeros((1, len(self.tokenizer.vocab)), dtype=torch.float32) + neg_inf
debug_list = []
for k, v in self.tokenizer.vocab.items(): for k, v in self.tokenizer.vocab.items():
if k in positive_words: if k in positive_words:
self.logits_bias[0, v] = 0 self.logits_bias[0, v] = 0
debug_list.append(k[1:])
print(f'Fooocus V2 Expansion: Vocab with {len(debug_list)} words.')
# debug_list = '\n'.join(sorted(debug_list))
# print(debug_list)
# t11 = self.tokenizer(',', return_tensors="np") # t11 = self.tokenizer(',', return_tensors="np")
# t198 = self.tokenizer('\n', return_tensors="np") # t198 = self.tokenizer('\n', return_tensors="np")