47 lines
1.1 KiB
Python
47 lines
1.1 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from tqdm import tqdm
|
|
|
|
# Necessary to load the local gguf package
|
|
if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from gguf.huggingface_hub import HFVocabRequest
|
|
|
|
logger = logging.getLogger("gguf-gen-pre")
|
|
|
|
|
|
def test_pre_tok(content) -> None:
|
|
pass
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("hf_auth_token", help="A huggingface read auth token")
|
|
parser.add_argument(
|
|
"-v", "--verbose", action="store_true", help="A huggingface read auth token"
|
|
)
|
|
parser.add_argument(
|
|
"-m", "--model-path", default=None, help="The models storage path"
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose:
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
else:
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
hf_vocab_req = HFVocabRequest(
|
|
args.model_path, args.hf_auth_token, logger
|
|
)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|