From 75d1abe24ac4509e9fc9faf5d14e26cde80141f3 Mon Sep 17 00:00:00 2001 From: Han Yin Date: Mon, 12 May 2025 15:44:22 -0700 Subject: [PATCH] [WIP] llama: ABI split builds five .so artifacts. However, all .so are performing on SVE level --- .../llama/src/main/cpp/CMakeLists.txt | 127 ++++++++---------- .../java/android/llama/cpp/LLamaAndroid.kt | 2 +- 2 files changed, 60 insertions(+), 69 deletions(-) diff --git a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt index e524a61df2..0f24e556f9 100644 --- a/examples/llama.android/llama/src/main/cpp/CMakeLists.txt +++ b/examples/llama.android/llama/src/main/cpp/CMakeLists.txt @@ -1,92 +1,83 @@ -# Sets the minimum CMake version required for this project. cmake_minimum_required(VERSION 3.22.1) - project("llama_android" VERSION 1.0.0 LANGUAGES C CXX) +# -------------------------------------------------------------------- +# 0. Global language & toolchain options +# -------------------------------------------------------------------- set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED true) - set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED true) -set(GGML_CPU_KLEIDIAI ON CACHE BOOL "" FORCE) -set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE) -set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE) -set(LLAMA_CURL OFF CACHE BOOL "" FORCE) - -string(APPEND CMAKE_C_FLAGS " -march=armv8.7-a+dotprod+i8mm+sve") -string(APPEND CMAKE_CXX_FLAGS " -march=armv8.7-a+dotprod+i8mm+sve") +## ---- ggml / llama feature switches (configure-time) ---------------- +#set(GGML_CPU_KLEIDIAI ON CACHE BOOL "" FORCE) +set(GGML_LLAMAFILE OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE) +set(LLAMA_CURL OFF CACHE BOOL "" FORCE) +# baseline ISA used for every source that is NOT in a tier wrapper +string(APPEND CMAKE_C_FLAGS " -march=armv9.2-a+dotprod+i8mm+sve+sve2+sme") +string(APPEND CMAKE_CXX_FLAGS " -march=armv9.2-a+dotprod+i8mm+sve+sve2+sme") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" CACHE STRING "" FORCE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "" FORCE) -## CMake definitions -#add_compile_definitions( -# GGML_CPU_KLEIDIAI=ON -# GGML_LLAMAFILE=OFF -# LLAMA_BUILD_COMMON=ON -# LLAMA_CURL=OFF -#) +# NOT NEEDED WHEN CMAKE_BUILD_TYPE PASSED IN BY GRADLE +#message(DEBUG "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") +#if(CMAKE_BUILD_TYPE STREQUAL "Debug") +# string(APPEND CMAKE_C_FLAGS " -O3 -DNDEBUG") +# string(APPEND CMAKE_CXX_FLAGS " -O3 -DNDEBUG") +#endif() -# Force Release only when Gradle didn't specify one +# default to Release if Gradle hasn’t set a variant if(NOT DEFINED CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) endif() -## Also provides "common" -#include(FetchContent) -#FetchContent_Declare( -# llama -# GIT_REPOSITORY https://github.com/ggml-org/llama.cpp -# GIT_TAG master -#) -#FetchContent_MakeAvailable(llama) +### Also provides "common" +##include(FetchContent) +##FetchContent_Declare( +## llama +## GIT_REPOSITORY https://github.com/ggml-org/llama.cpp +## GIT_TAG master +##) +##FetchContent_MakeAvailable(llama) -# Add local llama.cpp as subdirectory -add_subdirectory(../../../../../../ build-llama) +# -------------------------------------------------------------------- +# 1. Pull in llama.cpp (creates targets "llama" and "common") +# -------------------------------------------------------------------- +set(LLAMA_SRC ${CMAKE_CURRENT_LIST_DIR}/../../../../../../) +add_subdirectory(${LLAMA_SRC} build-llama) -#function(build_llama_tier tier march_flag) -# set(LIB_NAME "${CMAKE_PROJECT_NAME}_${tier}") -# -## set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8.7-a+i8mm") -## set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8.7-a+i8mm") -# -# add_library(${LIB_NAME} SHARED llama-android.cpp) -# +# ----------------------------------------------------------- +# 2. Helper to create one wrapper .so for a hardware tier +# ----------------------------------------------------------- +function(build_llama_tier tier march) + set(wrapper llama_android_${tier}) + add_library(${wrapper} SHARED llama-android.cpp) -# MIGHT NOT WORK???? -# target_include_directories(${LIB_NAME} PRIVATE -# ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../../common) -# ????? + # ISA tuning that belongs only to this tier + target_compile_options(${wrapper} PRIVATE "-march=${march}") -# target_compile_options(${LIB_NAME} PRIVATE "-march=${march_flag}") -# -# target_link_libraries(${LIB_NAME} -# llama -# common -# android -# log -# ) -# -# set_target_properties(${LIB_NAME} PROPERTIES -# OUTPUT_NAME "llama_android_${tier}" -# ) -#endfunction() -# -#build_llama_tier(t0 "armv8-a+simd") -#build_llama_tier(t1 "armv8.2-a+dotprod") -#build_llama_tier(t2 "armv8.6-a+dotprod+i8mm") -#build_llama_tier(t3 "armv9-a+dotprod+i8mm+sve+sve2") -#build_llama_tier(t4 "armv9.2-a+dotprod+i8mm+sve+sve2+sme") +# NOT NEEDED! +# # sampling.h lives in common/, ggml-impl.h lives in ggml/src +# target_include_directories(${wrapper} PRIVATE +# ${LLAMA_SRC}/common +# ${LLAMA_SRC}/ggml/src) + target_link_libraries(${wrapper} PRIVATE + llama + common + android + log) -add_library(${CMAKE_PROJECT_NAME} SHARED - # List C/C++ source files with relative paths to this CMakeLists.txt. - llama-android.cpp) + set_target_properties(${wrapper} PROPERTIES OUTPUT_NAME ${wrapper}) +endfunction() -target_link_libraries(${CMAKE_PROJECT_NAME} - # List libraries link to the target library - llama - common - android - log) +# ----------------------------------------------------------- +# 3. Build the five tiers (no KleidiAI for now) +# ----------------------------------------------------------- +build_llama_tier(t0 "armv8-a+simd") +build_llama_tier(t1 "armv8.2-a+dotprod") +build_llama_tier(t2 "armv8.6-a+dotprod+i8mm") +build_llama_tier(t3 "armv9-a+dotprod+i8mm+sve+sve2") +build_llama_tier(t4 "armv9.2-a+dotprod+i8mm+sve+sve2+sme") diff --git a/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt index 32f81deedf..0b6dad29e7 100644 --- a/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt +++ b/examples/llama.android/llama/src/main/java/android/llama/cpp/LLamaAndroid.kt @@ -240,7 +240,7 @@ class LLamaAndroid private constructor() : InferenceEngine { private val TAG = LLamaAndroid::class.simpleName // TODO-han.yin: replace with dynamic loader - private const val LIB_LLAMA_ANDROID = "llama_android" + private const val LIB_LLAMA_ANDROID = "llama_android_t4" // Enforce only one instance of Llm. private val _instance: LLamaAndroid = LLamaAndroid()