happyz
/
llama.cpp
mirror of https://github.com/ggerganov/llama.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

llama : split unicodedata.cpp from unicode.cpp

This commit is contained in:
Jared Van Bortel 2024-03-25 16:00:03 -04:00
parent b80c0af078
commit e5ddf2fcdd
5 changed files with 1430 additions and 1404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -1165,6 +1165,7 @@ add_library(llama
llama.h
unicode.h
unicode.cpp
unicodedata.cpp
)
target_include_directories(llama PUBLIC .)

2
Makefile
View File

@ -663,7 +663,7 @@ ggml-backend.o: ggml-backend.c ggml.h ggml-backend.h
ggml-quants.o: ggml-quants.c ggml.h ggml-quants.h ggml-common.h
$(CC) $(CFLAGS) -c $< -o $@
unicode.o: unicode.cpp unicode.h
unicode.o: unicode.cpp unicodedata.cpp unicode.h unicodedata.h
$(CXX) $(CXXFLAGS) -c $< -o $@
OBJS += ggml-alloc.o ggml-backend.o ggml-quants.o unicode.o

1407
unicode.cpp
View File

File diff suppressed because it is too large Load Diff

1409
unicodedata.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

15
unicodedata.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include <cstdint>
#include <map>
#include <utility>
#include <vector>
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_digit;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_letter;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_whitespace;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_accent_mark;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_punctuation;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_symbol;
extern const std::vector<std::pair<uint32_t, uint32_t>> unicode_ranges_control;
extern const std::multimap<uint32_t, uint32_t> unicode_map_nfd;