#pragma once #include #include #include #include #include // header file of Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK // https://qpm.qualcomm.com/#/main/tools/details/qualcomm_ai_engine_direct #include "QnnTypes.h" #include "QnnCommon.h" #include "QnnInterface.h" #include "QnnContext.h" #include "QnnBackend.h" #include "QnnGraph.h" #include "QnnProperty.h" #include "QnnTensor.h" #include "System/QnnSystemInterface.h" #include "HTP/QnnHtpDevice.h" #include "HTP/QnnHtpGraph.h" #include "qnn-types.hpp" #include "utils.hpp" namespace qnn { // ================================================================================================= // // wrapper class of Qualcomm QNN(Qualcomm Neural Network, aka Qualcomm AI Engine Direct) SDK // ref:https://github.com/pytorch/executorch/tree/main/backends/qualcomm // ================================================================================================= class qnn_interface { #define DEFINE_SHIM_FUNCTION_INTERFACE(F, pointer_name) \ template inline auto qnn_##F(Args... args) const { \ return (_qnn_interface->QNN_INTERFACE_VER_NAME.pointer_name)( \ std::forward(args)...); \ } #define DEFINE_SHIM_FUNCTION_SYS_INTERFACE(F, pointer_name) \ template inline auto qnn_##F(Args... args) const { \ return ( \ _qnn_sys_interface->QNN_SYSTEM_INTERFACE_VER_NAME.pointer_name)( \ std::forward(args)...); \ } friend class qnn_instance; public: qnn_interface() = default; // QnnBackend DEFINE_SHIM_FUNCTION_INTERFACE(backend_create, backendCreate); DEFINE_SHIM_FUNCTION_INTERFACE(backend_free, backendFree); DEFINE_SHIM_FUNCTION_INTERFACE(backend_register_op_package, backendRegisterOpPackage); DEFINE_SHIM_FUNCTION_INTERFACE(backend_validate_op_config, backendValidateOpConfig); DEFINE_SHIM_FUNCTION_INTERFACE(backend_get_api_version, backendGetApiVersion); // QnnDevice DEFINE_SHIM_FUNCTION_INTERFACE(device_create, deviceCreate); DEFINE_SHIM_FUNCTION_INTERFACE(device_free, deviceFree); DEFINE_SHIM_FUNCTION_INTERFACE(device_get_infrastructure, deviceGetInfrastructure); DEFINE_SHIM_FUNCTION_INTERFACE(device_get_platform_info, deviceGetPlatformInfo); DEFINE_SHIM_FUNCTION_INTERFACE(device_get_info, deviceGetInfo); // QnnContext DEFINE_SHIM_FUNCTION_INTERFACE(context_create, contextCreate); DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary_size, contextGetBinarySize); DEFINE_SHIM_FUNCTION_INTERFACE(context_get_binary, contextGetBinary); DEFINE_SHIM_FUNCTION_INTERFACE(context_create_from_binary, contextCreateFromBinary); DEFINE_SHIM_FUNCTION_INTERFACE(context_free, contextFree); // QnnGraph DEFINE_SHIM_FUNCTION_INTERFACE(graph_create, graphCreate); DEFINE_SHIM_FUNCTION_INTERFACE(graph_add_node, graphAddNode); DEFINE_SHIM_FUNCTION_INTERFACE(graph_finalize, graphFinalize); DEFINE_SHIM_FUNCTION_INTERFACE(graph_execute, graphExecute); DEFINE_SHIM_FUNCTION_INTERFACE(graph_retrieve, graphRetrieve); // QnnLog DEFINE_SHIM_FUNCTION_INTERFACE(log_create, logCreate); DEFINE_SHIM_FUNCTION_INTERFACE(log_free, logFree); DEFINE_SHIM_FUNCTION_INTERFACE(log_set_log_level, logSetLogLevel); // QnnProfile DEFINE_SHIM_FUNCTION_INTERFACE(profile_create, profileCreate); DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_events, profileGetEvents); DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_sub_events, profileGetSubEvents); DEFINE_SHIM_FUNCTION_INTERFACE(profile_get_event_data, profileGetEventData); DEFINE_SHIM_FUNCTION_INTERFACE(profile_free, profileFree); // QnnMem DEFINE_SHIM_FUNCTION_INTERFACE(mem_register, memRegister); DEFINE_SHIM_FUNCTION_INTERFACE(mem_de_register, memDeRegister); // QnnProperty DEFINE_SHIM_FUNCTION_INTERFACE(property_has_capability, propertyHasCapability); // QnnTensor DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_context_tensor, tensorCreateContextTensor); DEFINE_SHIM_FUNCTION_INTERFACE(tensor_create_graph_tensor, tensorCreateGraphTensor); // QnnSystem DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_create, systemContextCreate); DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_get_binary_info, systemContextGetBinaryInfo); DEFINE_SHIM_FUNCTION_SYS_INTERFACE(system_context_free, systemContextFree); void set_qnn_interface(const QnnInterface_t* qnn_interface) { _qnn_interface = qnn_interface; } void set_qnn_system_interface( const QnnSystemInterface_t* qnn_sys_interface) { _qnn_sys_interface = qnn_sys_interface; } uint32_t get_backend_id() const { return _qnn_interface->backendId; } bool is_loaded() const { return ((_qnn_sys_interface != nullptr) && (_qnn_interface != nullptr)); } private: const QnnInterface_t* _qnn_interface = nullptr; const QnnSystemInterface_t* _qnn_sys_interface = nullptr; }; class qnn_instance { public: using BackendIdType = decltype(QnnInterface_t{}.backendId); explicit qnn_instance(const std::string& lib_path, const std::string& backend_name, const std::string& model_name) : _lib_path(std::move(lib_path)) , _backend_name(std::move(backend_name)) , _model_name(std::move(model_name)) {}; ~qnn_instance() {} int qnn_init(const QnnSaver_Config_t** saver_config) { BackendIdType backend_id = QNN_BACKEND_ID_NULL; QNN_LOG_DEBUG("enter qni_init\n"); std::lock_guard lock(_init_mutex); if (0 != load_system()) { QNN_LOG_WARN("can not load QNN system lib, pls check why?\n"); return 1; } else { QNN_LOG_DEBUG("load QNN system lib successfully\n"); } std::string backend_lib_path = _lib_path + _backend_name; if (0 == _lib_path_to_backend_id.count(backend_lib_path)) { int is_load_ok = load_backend(backend_lib_path, saver_config); if (0 != is_load_ok) { QNN_LOG_WARN("failed to load QNN backend\n"); return 2; } } backend_id = _lib_path_to_backend_id[backend_lib_path]; if (0 == _loaded_backend.count(backend_id) || 0 == _loaded_lib_handle.count(backend_id)) { QNN_LOG_WARN("library %s is loaded but loaded backend count=%zu, " "loaded lib_handle count=%zu\n", backend_lib_path.c_str(), _loaded_backend.count(backend_id), _loaded_lib_handle.count(backend_id)); return 3; } _qnn_interface.set_qnn_interface(_loaded_backend[backend_id]); _qnn_interface.qnn_log_create(qnn::sdk_logcallback, _qnn_log_level, &_qnn_log_handle); if (nullptr == _qnn_log_handle) { // NPU backend not work on Qualcomm SoC equipped low-end phone QNN_LOG_WARN("why failed to initialize qnn log\n"); return 4; } else { QNN_LOG_DEBUG("initialize qnn log successfully\n"); } std::vector temp_backend_config; _qnn_interface.qnn_backend_create( _qnn_log_handle, temp_backend_config.empty() ? nullptr : temp_backend_config.data(), &_qnn_backend_handle); if (nullptr == _qnn_backend_handle) { QNN_LOG_WARN("why failed to initialize qnn backend\n"); return 5; } else { QNN_LOG_DEBUG("initialize qnn backend successfully\n"); } if (nullptr != _qnn_raw_interface.propertyHasCapability) { Qnn_ErrorHandle_t qnn_status = _qnn_raw_interface.propertyHasCapability(QNN_PROPERTY_GROUP_DEVICE); if (QNN_PROPERTY_NOT_SUPPORTED == qnn_status) { QNN_LOG_WARN("device property is not supported\n"); } if (QNN_PROPERTY_ERROR_UNKNOWN_KEY == qnn_status) { QNN_LOG_WARN("device property is not known to backend\n"); } } Qnn_ErrorHandle_t qnn_status = QNN_SUCCESS; if (_backend_name.find("Htp") != std::variant_npos) { const QnnDevice_PlatformInfo_t* p_info = nullptr; _qnn_raw_interface.deviceGetPlatformInfo(nullptr, &p_info); QNN_LOG_INFO("device counts %d", p_info->v1.numHwDevices); QnnDevice_HardwareDeviceInfo_t* infos = p_info->v1.hwDevices; QnnHtpDevice_OnChipDeviceInfoExtension_t chipinfo = { }; for (int i = 0; i < p_info->v1.numHwDevices; i++) { QNN_LOG_INFO("deviceID:%d, deviceType:%d, numCores %d", infos[i].v1.deviceId, infos[i].v1.deviceType, infos[i].v1.numCores); QnnDevice_DeviceInfoExtension_t devinfo = infos[i].v1.deviceInfoExtension; chipinfo = devinfo->onChipDevice; QnnHtpDevice_Arch_t htp_arch = chipinfo.arch; QNN_LOG_INFO("htp_type:%d(%s)", devinfo->devType, (devinfo->devType == QNN_HTP_DEVICE_TYPE_ON_CHIP) ? "ON_CHIP" : ""); QNN_LOG_INFO("qualcomm soc_model:%d(%s), htp_arch:%d(%s), vtcm_size:%d MB", chipinfo.socModel, qnn::get_chipset_desc(chipinfo.socModel), htp_arch, qnn::get_htparch_desc(htp_arch), chipinfo.vtcmSize); _soc_info = { chipinfo.socModel, htp_arch, chipinfo.vtcmSize }; } _qnn_raw_interface.deviceFreePlatformInfo(nullptr, p_info); QnnHtpDevice_CustomConfig_t soc_customconfig; soc_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_SOC; soc_customconfig.socModel = chipinfo.socModel; QnnDevice_Config_t soc_devconfig; soc_devconfig.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM; soc_devconfig.customConfig = &soc_customconfig; QnnHtpDevice_CustomConfig_t arch_customconfig; arch_customconfig.option = QNN_HTP_DEVICE_CONFIG_OPTION_ARCH; arch_customconfig.arch.arch = chipinfo.arch; arch_customconfig.arch.deviceId = 0; // Id of device to be used. If single device is used by default 0. QnnDevice_Config_t arch_devconfig; arch_devconfig.option = QNN_DEVICE_CONFIG_OPTION_CUSTOM; arch_devconfig.customConfig = &arch_customconfig; const QnnDevice_Config_t* p_deviceconfig[] = { &soc_devconfig, &arch_devconfig, nullptr }; qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, p_deviceconfig, &_qnn_device_handle); } else { qnn_status = _qnn_raw_interface.deviceCreate(_qnn_log_handle, nullptr, &_qnn_device_handle); } if (QNN_SUCCESS != qnn_status && QNN_DEVICE_ERROR_UNSUPPORTED_FEATURE != qnn_status) { QNN_LOG_WARN("failed to create QNN device\n"); } else { QNN_LOG_INFO("create QNN device successfully\n"); } if (qnn::sdk_profile_level::profile_off != _profile_level) { QNN_LOG_INFO("profiling turned on; level = %d", _profile_level); if (qnn::sdk_profile_level::profile_basic == _profile_level) { QNN_LOG_INFO("basic profiling requested. creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(_qnn_backend_handle, QNN_PROFILE_LEVEL_BASIC, &_qnn_profile_handle)) { QNN_LOG_WARN("unable to create profile handle in the backend\n"); return 6; } else { QNN_LOG_DEBUG("initialize qnn profile successfully\n"); } } else if (qnn::sdk_profile_level::profile_detail == _profile_level) { QNN_LOG_INFO("detailed profiling requested. Creating Qnn Profile object\n"); if (QNN_PROFILE_NO_ERROR != _qnn_raw_interface.profileCreate(_qnn_backend_handle, QNN_PROFILE_LEVEL_DETAILED, &_qnn_profile_handle)) { QNN_LOG_WARN("unable to create profile handle in the backend\n"); return 7; } else { QNN_LOG_DEBUG("initialize qnn profile successfully\n"); } } } _rpc_lib_handle = dlopen("libcdsprpc.so", RTLD_NOW | RTLD_LOCAL); if (nullptr == _rpc_lib_handle) { QNN_LOG_WARN("failed to load qualcomm's rpc lib, error:%s\n", dlerror()); return 8; } else { QNN_LOG_DEBUG("load rpcmem lib successfully\n"); set_rpcmem_initialized(true); } _pfn_rpc_mem_init = reinterpret_cast( dlsym(_rpc_lib_handle, "rpcmem_init")); _pfn_rpc_mem_deinit = reinterpret_cast( dlsym(_rpc_lib_handle, "rpcmem_deinit")); _pfn_rpc_mem_alloc = reinterpret_cast( dlsym(_rpc_lib_handle, "rpcmem_alloc")); _pfn_rpc_mem_free = reinterpret_cast( dlsym(_rpc_lib_handle, "rpcmem_free")); _pfn_rpc_mem_to_fd = reinterpret_cast( dlsym(_rpc_lib_handle, "rpcmem_to_fd")); if (nullptr == _pfn_rpc_mem_alloc || nullptr == _pfn_rpc_mem_free || nullptr == _pfn_rpc_mem_to_fd) { QNN_LOG_WARN("unable to access symbols in QNN RPC lib. dlerror(): %s", dlerror()); dlclose(_rpc_lib_handle); return 9; } if (nullptr != _pfn_rpc_mem_init) { // make Qualcomm's SoC equipped low-end phone happy _pfn_rpc_mem_init(); } /* TODO: not used, keep it for further usage QnnContext_Config_t qnn_context_config = QNN_CONTEXT_CONFIG_INIT; qnn_context_config.priority = QNN_PRIORITY_DEFAULT; const QnnContext_Config_t * context_configs[] = {&qnn_context_config, nullptr}; */ _qnn_interface.qnn_context_create( _qnn_backend_handle, _qnn_device_handle, nullptr, &_qnn_context_handle); if (nullptr == _qnn_context_handle) { QNN_LOG_WARN("why failed to initialize qnn context\n"); return 10; } else { QNN_LOG_DEBUG("initialize qnn context successfully\n"); } if (_backend_name.find("Htp") != std::variant_npos) { //TODO: faster approach to probe the accurate capacity of rpc ion memory size_t candidate_size = 0; uint8_t* rpc_buffer = nullptr; const int size_in_mb = (1 << 20); size_t probe_slots[] = { 1024, 1536, 2048 - 48, 2048 }; size_t probe_counts = sizeof(probe_slots) / sizeof(size_t); for (size_t idx = 0; idx < probe_counts; idx++) { rpc_buffer = static_cast(alloc_rpcmem( probe_slots[idx] * size_in_mb, 4)); if (nullptr == rpc_buffer) { QNN_LOG_INFO("alloc rpcmem %d (MB) failure, %s\n", probe_slots[idx], strerror(errno)); break; } else { candidate_size = probe_slots[idx]; free_rpcmem(rpc_buffer); rpc_buffer = nullptr; } } if (candidate_size > _rpcmem_capacity) _rpcmem_capacity = candidate_size; QNN_LOG_INFO("capacity of QNN rpc ion memory is about %d MB\n", _rpcmem_capacity); if (0 != init_htp_perfinfra()) { QNN_LOG_WARN("initialize HTP performance failure"); } if (0 != set_rpc_polling()) { QNN_LOG_WARN("set RPC polling failure"); } if (0 != set_high_performance_mode()) { QNN_LOG_WARN("set HTP high performance mode failure"); } } QNN_LOG_DEBUG("leave qni_init\n"); return 0; } int qnn_finalize() { int ret_status = 0; Qnn_ErrorHandle_t error = QNN_SUCCESS; if (nullptr != _pfn_rpc_mem_deinit) // make Qualcomm's SoC equipped low-end phone happy _pfn_rpc_mem_deinit(); if (dlclose(_rpc_lib_handle) != 0) { QNN_LOG_WARN("failed to unload qualcomm's rpc lib, error:%s\n", dlerror()); } else { QNN_LOG_DEBUG("succeed to close rpcmem lib\n"); } if (_backend_name.find("Htp") != std::variant_npos) { _qnn_htp_perfinfra->destroyPowerConfigId(_qnn_power_configid); } if (nullptr != _qnn_context_handle) { error = _qnn_interface.qnn_context_free(_qnn_context_handle, _qnn_profile_handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN context_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); } _qnn_context_handle = nullptr; } if (nullptr != _qnn_profile_handle) { error = _qnn_interface.qnn_profile_free(_qnn_profile_handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN profile_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); } _qnn_profile_handle = nullptr; } if (nullptr != _qnn_device_handle) { error = _qnn_interface.qnn_device_free(_qnn_device_handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN device_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); } _qnn_device_handle = nullptr; } if (nullptr != _qnn_backend_handle) { error = _qnn_interface.qnn_backend_free(_qnn_backend_handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN backend_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); } _qnn_backend_handle = nullptr; } if (nullptr != _qnn_log_handle) { error = _qnn_interface.qnn_log_free(_qnn_log_handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN log_handle: ID %u, error %d\n", _qnn_interface.get_backend_id(), QNN_GET_ERROR_CODE(error)); } _qnn_log_handle = nullptr; } unload_backend(); unload_system(); return ret_status; } //TODO:keep it for further usage of offload the entire cgraph to a single QNN DAG directly // which was used in Qualcomm's dedicated AI technology #if 0 int init_qnn_graph(const char* graph_name, bool debug, uint8_t do_node_validation = true, const QnnGraph_Config_t** graph_configs = nullptr) { int result = 0; if (nullptr == graph_name) { QNN_LOG_WARN("graph name is null\n"); return 1; } if (!_graph_name.empty()) { QNN_LOG_WARN("qnn model for graph %s already initialized\n", graph_name); return 2; } if (!do_node_validation) { QNN_LOG_WARN("node validation disabled, backend will not perform op " "validation prior to adding node\n"); } _graph_name = graph_name; _debug_tensor = debug; _do_node_validations = do_node_validation; result = _qnn_raw_interface.graphCreate(_qnn_context_handle, graph_name, graph_configs, &_qnn_graph_handle); if (result != QNN_GRAPH_NO_ERROR || nullptr == _qnn_graph_handle) { QNN_LOG_WARN("failed to create graph in qnn context\n"); return 3; } else { QNN_LOG_INFO("succeed to create graph %s, %p\n", graph_name, _qnn_graph_handle); } return 0; } int finalize_qnn_graph() { if (nullptr != _qnn_graph_handle) { if (_qnn_raw_interface.graphFinalize(_qnn_graph_handle, _qnn_profile_handle, nullptr) != QNN_GRAPH_NO_ERROR) { QNN_LOG_WARN("finalizing graph failure\n"); } } else { QNN_LOG_DEBUG("qnn graph handle is null\n"); } return 0; } #endif const qnn_interface& get_qnn_interface() { if (!_qnn_interface.is_loaded()) { QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_interface; } const QNN_INTERFACE_VER_TYPE& get_qnn_raw_interface() { if (!_qnn_interface.is_loaded()) { QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_raw_interface; } const QNN_SYSTEM_INTERFACE_VER_TYPE& get_qnn_raw_system_interface() { if (!_qnn_interface.is_loaded()) { QNN_LOG_WARN("pls check why _qnn_interface is not loaded\n"); } return _qnn_raw_system_interface; } const Qnn_LogHandle_t get_qnn_log_handle() { return _qnn_log_handle; } const Qnn_ProfileHandle_t get_qnn_profile_handle() { return _qnn_profile_handle; } const Qnn_DeviceHandle_t get_qnn_device_handle() { return _qnn_device_handle; } const Qnn_BackendHandle_t get_qnn_backend_handle() { return _qnn_backend_handle; } const Qnn_ContextHandle_t get_qnn_context_handle() { return _qnn_context_handle; } const QnnSystemContext_Handle_t get_qnn_system_handle() { return _qnn_system_handle; } const Qnn_GraphHandle_t get_qnn_graph_handle() { return _qnn_graph_handle; } int init_htp_perfinfra() { QnnDevice_Infrastructure_t device_infra = nullptr; int error = _qnn_raw_interface.deviceGetInfrastructure(&device_infra); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to get qnn device infra\n"); return 1; } else { QNN_LOG_INFO("HTP backend perf_infrastructure creation ok\n"); } QnnHtpDevice_Infrastructure_t* htp_infra = static_cast(device_infra); QnnHtpDevice_PerfInfrastructure_t* htp_perfinfra = &htp_infra->perfInfra; uint32_t power_configid = 1; uint32_t device_id = 0; uint32_t core_id = 0; htp_perfinfra->createPowerConfigId(device_id, core_id, &power_configid); if (htp_infra->infraType != QNN_HTP_DEVICE_INFRASTRUCTURE_TYPE_PERF) { QNN_LOG_INFO("HTP infra type = %d, which is not perf infra type", htp_infra->infraType); } else { QNN_LOG_INFO("HTP infra type = %d, which is perf infra type\n", htp_infra->infraType); } _qnn_htp_perfinfra = htp_perfinfra; _qnn_power_configid = power_configid; return 0; } int set_rpc_polling() { if (_qnn_htp_perfinfra) { QnnHtpPerfInfrastructure_PowerConfig_t rpc_polling_time; memset(&rpc_polling_time, 0, sizeof(rpc_polling_time)); rpc_polling_time.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME; //use rpc polling time recommended 0-10000 us rpc_polling_time.rpcPollingTimeConfig = 9999; QnnHtpPerfInfrastructure_PowerConfig_t rpc_control_latency; memset(&rpc_control_latency, 0, sizeof(rpc_control_latency)); rpc_control_latency.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY; //use rpc control latency recommended 100 us, refer hexagon sdk rpc_control_latency.rpcControlLatencyConfig = 100; const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = { &rpc_polling_time, &rpc_control_latency, nullptr }; Qnn_ErrorHandle_t qnn_status = _qnn_htp_perfinfra->setPowerConfig( _qnn_power_configid, power_configs); if (qnn_status != QNN_SUCCESS) { QNN_LOG_WARN("set htp perf failed\n"); } else { QNN_LOG_INFO("set htp perf ok\n"); } } else { QNN_LOG_WARN("can't set htp perf\n"); } return 0; } int set_high_performance_mode() { if (nullptr == _qnn_htp_perfinfra) { QNN_LOG_WARN("perf intra is null\n"); return 1; } QnnHtpPerfInfrastructure_PowerConfig_t power_config; memset(&power_config, 0, sizeof(power_config)); power_config.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_DCVS_V3; power_config.dcvsV3Config.setDcvsEnable = 1; power_config.dcvsV3Config.dcvsEnable = 0; power_config.dcvsV3Config.contextId = _qnn_power_configid; power_config.dcvsV3Config.powerMode = QNN_HTP_PERF_INFRASTRUCTURE_POWERMODE_PERFORMANCE_MODE; power_config.dcvsV3Config.setSleepLatency = 1; // true to consider Latency parameter otherwise false power_config.dcvsV3Config.sleepLatency = 40; power_config.dcvsV3Config.setBusParams = 1; // true to consider Bus parameter otherwise false power_config.dcvsV3Config.setCoreParams = 1; // true to consider Core parameter otherwise false power_config.dcvsV3Config.sleepDisable = 1; // true to consider sleep/LPM modes, false to enable power_config.dcvsV3Config.setSleepDisable = 1; // true to consider sleep disable/enable parameter otherwise false set sleep latency parameter // set Bus Clock Parameters power_config.dcvsV3Config.busVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.busVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.busVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; // set Core Clock Parameters power_config.dcvsV3Config.coreVoltageCornerMin = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.coreVoltageCornerTarget = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; power_config.dcvsV3Config.coreVoltageCornerMax = DCVS_VOLTAGE_VCORNER_MAX_VOLTAGE_CORNER; // set power config with different performance parameters const QnnHtpPerfInfrastructure_PowerConfig_t* power_configs[] = { &power_config, nullptr }; Qnn_ErrorHandle_t qnn_status = QNN_SUCCESS; qnn_status = _qnn_htp_perfinfra->setPowerConfig(_qnn_power_configid, power_configs); if (qnn_status != QNN_SUCCESS) { QNN_LOG_WARN("set htp high performance mode failed\n"); } else { QNN_LOG_INFO("set htp high performance mode ok\n"); } return 0; } std::string& get_qnn_graph_name() { return _graph_name; } bool is_rpcmem_initialized() { return _rpcmem_initialized; } void set_rpcmem_initialized(bool initialized) { _rpcmem_initialized = initialized; } size_t get_rpcmem_capacity() { return _rpcmem_capacity; } bool is_rpcmem_registered(Qnn_MemHandle_t handle) { return _qnn_mem_set.count(handle) != 0U; } void* alloc_rpcmem(size_t bytes, size_t alignment) { if (!_rpcmem_initialized) { QNN_LOG_WARN("rpc memory not initialized\n"); return nullptr; } auto allocate_bytes = static_cast(bytes + alignment); void* buf = _pfn_rpc_mem_alloc(RPCMEM_HEAP_ID_SYSTEM, RPCMEM_DEFAULT_FLAGS, allocate_bytes); if (buf == nullptr) { QNN_LOG_WARN("failed to allocate rpc memory\n"); return nullptr; } auto aligned_buf = reinterpret_cast( qnn::align_to(alignment, reinterpret_cast(buf))); bool status = _rpcmem_store_map.insert(std::pair(aligned_buf, buf)).second; if (!status) { QNN_LOG_WARN("failed to allocate rpc memory\n"); _pfn_rpc_mem_free(buf); } return aligned_buf; } void free_rpcmem(void* buf) { if (!_rpcmem_initialized) { QNN_LOG_WARN("rpc memory not initialized\n"); } else if (0 == _rpcmem_store_map.count(buf)) { QNN_LOG_WARN("no allocated tensor\n"); } else { _pfn_rpc_mem_free(_rpcmem_store_map[buf]); _rpcmem_store_map.erase(buf); } } int32_t rpcmem_to_fd(void* buf) { int32_t mem_fd = -1; if (!is_rpcmem_initialized()) { QNN_LOG_WARN("rpc memory not initialized\n"); } else { mem_fd = _pfn_rpc_mem_to_fd(buf); } return mem_fd; } int register_rpcmem(void* p_data, Qnn_Tensor_t* p_tensor) { if (nullptr == p_data || (nullptr == p_tensor)) { QNN_LOG_WARN("invalid param\n"); return 1; } if (!is_rpcmem_initialized()) { QNN_LOG_WARN("rpc memory not initialized\n"); return 2; } if (is_rpcmem_allocated(p_data)) { QNN_LOG_WARN("rpc memory already allocated\n"); return 3; } if (is_rpcmem_registered((QNN_VER_PTR(*p_tensor)->memHandle))) { QNN_LOG_WARN("tensor %s has been registered shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); return 4; } int32_t mem_fd = rpcmem_to_fd(p_data); if (-1 == mem_fd) { QNN_LOG_WARN("failed to get file descriptor\n"); return 5; } QNN_LOG_INFO("mem_fd %d\n", mem_fd); Qnn_MemDescriptor_t descriptor = { {QNN_VER_PTR(*p_tensor)->rank, QNN_VER_PTR(*p_tensor)->dimensions, nullptr}, QNN_VER_PTR(*p_tensor)->dataType, QNN_MEM_TYPE_ION, {{mem_fd}} }; Qnn_MemHandle_t handle = nullptr; int error = QNN_SUCCESS; error = _qnn_interface.qnn_mem_register(_qnn_context_handle, &descriptor, /*numDescriptors=*/1, &handle); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to register shared memory, error %d, %s\n", QNN_GET_ERROR_CODE(error), strerror(error)); return 6; } else { QNN_LOG_INFO("tensor %s successfully register shared memory\n", (QNN_VER_PTR(*p_tensor)->name)); } QNN_VER_PTR(*p_tensor)->memHandle = handle; _qnn_mem_set.insert((std::pair(p_data, handle))); return 0; } void* get_rpcmem_from_memhandle(Qnn_MemHandle_t mem_handle) { for (std::unordered_map::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end(); it++) { Qnn_MemHandle_t mem_handle = it->second; if (it->second == mem_handle) { return it->first; } } QNN_LOG_WARN("can't find rpcmem from qnn mem handle %p", mem_handle); return nullptr; } void unregister_rpcmem() { Qnn_ErrorHandle_t error = QNN_SUCCESS; if (_qnn_mem_set.empty()) { QNN_LOG_WARN("no rpcmem registered\n"); } for (std::unordered_map::iterator it = _qnn_mem_set.begin(); it != _qnn_mem_set.end(); it++) { Qnn_MemHandle_t mem_handle = it->second; error = _qnn_interface.qnn_mem_de_register(&mem_handle, 1); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to unregister shared memory, error %d\n", QNN_GET_ERROR_CODE(error)); } } _qnn_mem_set.clear(); } bool is_rpcmem_allocated(void* buf) { return _qnn_mem_set.count(buf) != 0U; } const qnn::qcom_socinfo& get_soc_info() { return _soc_info; } public: std::map> _qnn_graph_map; private: int load_system() { Qnn_ErrorHandle_t error = QNN_SUCCESS; std::string system_lib_path = _lib_path + "libQnnSystem.so"; QNN_LOG_DEBUG("system_lib_path:%s\n", system_lib_path.c_str()); _system_lib_handle = dlopen(system_lib_path.c_str(), RTLD_NOW | RTLD_LOCAL); if (nullptr == _system_lib_handle) { QNN_LOG_WARN("can not open QNN library %s, error: %s\n", system_lib_path.c_str(), dlerror()); return 1; } auto* get_providers = reinterpret_cast( dlsym(_system_lib_handle, "QnnSystemInterface_getProviders")); if (nullptr == get_providers) { QNN_LOG_WARN( "can not load QNN symbol QnnSystemInterface_getProviders: %s\n", dlerror()); return 2; } uint32_t num_providers = 0; const QnnSystemInterface_t** provider_list = nullptr; error = get_providers(&provider_list, &num_providers); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to get providers, error %d\n", QNN_GET_ERROR_CODE(error)); return 3; } if (num_providers != _required_num_providers) { QNN_LOG_WARN("providers is %d instead of required %d\n", num_providers, _required_num_providers); return 4; } if (nullptr == provider_list) { QNN_LOG_WARN("can not get providers\n"); return 5; } QNN_SYSTEM_INTERFACE_VER_TYPE qnn_system_interface; bool found_valid_system_interface = false; for (size_t idx = 0; idx < num_providers; idx++) { if (QNN_SYSTEM_API_VERSION_MAJOR == provider_list[idx]->systemApiVersion.major && QNN_SYSTEM_API_VERSION_MINOR <= provider_list[idx]->systemApiVersion.minor) { found_valid_system_interface = true; qnn_system_interface = provider_list[idx]->QNN_SYSTEM_INTERFACE_VER_NAME; break; } } if (!found_valid_system_interface) { QNN_LOG_WARN("unable to find a valid qnn system interface\n"); return 6; } else { QNN_LOG_INFO("find a valid qnn system interface\n"); } set_qnn_raw_system_interface(qnn_system_interface); _qnn_interface.set_qnn_system_interface(provider_list[0]); _qnn_interface.qnn_system_context_create(&_qnn_system_handle); if (nullptr == _qnn_system_handle) { QNN_LOG_WARN("can not create QNN system contenxt\n"); } else { QNN_LOG_INFO("initialize qnn system successfully\n"); } return 0; } int unload_system() { int result = 0; if (nullptr == _system_lib_handle) { QNN_LOG_WARN("system lib handle is null\n"); return 1; } if (nullptr != _qnn_system_handle) { result = _qnn_interface.qnn_system_context_free(_qnn_system_handle); if (result != QNN_SUCCESS) { QNN_LOG_WARN("failed to free QNN system context\n"); } _qnn_system_handle = nullptr; } int dlclose_error = dlclose(_system_lib_handle); if (dlclose_error != 0) { QNN_LOG_WARN("failed to close QnnSystem library, error %s\n", dlerror()); return 2; } _system_lib_handle = nullptr; return result; } int load_backend(std::string& lib_path, const QnnSaver_Config_t** saver_config) { Qnn_ErrorHandle_t error = QNN_SUCCESS; QNN_LOG_DEBUG("lib_path:%s\n", lib_path.c_str()); void* lib_handle = dlopen(lib_path.c_str(), RTLD_NOW | RTLD_GLOBAL); if (nullptr == lib_handle) { QNN_LOG_WARN("can not open QNN library %s, with error: %s", lib_path.c_str(), dlerror()); return 1; } auto get_providers = qnn::load_qnn_functionpointers( lib_handle, "QnnInterface_getProviders"); if (nullptr == get_providers) { QNN_LOG_WARN("can not load symbol QnnInterface_getProviders : %s", dlerror()); return 2; } std::uint32_t num_providers = 0; const QnnInterface_t** provider_list = nullptr; error = get_providers(&provider_list, &num_providers); if (error != QNN_SUCCESS) { QNN_LOG_WARN("failed to get providers, error %d", QNN_GET_ERROR_CODE(error)); return 3; } QNN_LOG_DEBUG("num_providers=%d\n", num_providers); if (num_providers != _required_num_providers) { QNN_LOG_WARN("providers is %d instead of required %d", num_providers, _required_num_providers); return 4; } if (nullptr == provider_list) { QNN_LOG_WARN("failed to get qnn interface providers\n"); return 5; } bool found_valid_interface = false; QNN_INTERFACE_VER_TYPE qnn_interface; for (size_t idx = 0; idx < num_providers; idx++) { if (QNN_API_VERSION_MAJOR == provider_list[idx]->apiVersion.coreApiVersion.major && QNN_API_VERSION_MINOR <= provider_list[idx]->apiVersion.coreApiVersion.minor) { found_valid_interface = true; qnn_interface = provider_list[idx]->QNN_INTERFACE_VER_NAME; break; } } if (!found_valid_interface) { QNN_LOG_WARN("unable to find a valid qnn interface\n"); return 6; } else { QNN_LOG_INFO("find a valid qnn interface\n"); } set_qnn_raw_interface(qnn_interface); BackendIdType backend_id = provider_list[0]->backendId; _lib_path_to_backend_id[lib_path] = backend_id; if (_loaded_backend.count(backend_id) > 0) { QNN_LOG_WARN("lib_path %s is loaded, but backend %d already exists\n", lib_path.c_str(), backend_id); } _loaded_backend[backend_id] = provider_list[0]; if (_loaded_lib_handle.count(backend_id) > 0) { QNN_LOG_WARN("closing %p\n", _loaded_lib_handle[backend_id]); int dlclose_error = dlclose(_loaded_lib_handle[backend_id]); if (dlclose_error != 0) { QNN_LOG_WARN("fail to close %p with error %s\n", _loaded_lib_handle[backend_id], dlerror()); } } _loaded_lib_handle[backend_id] = lib_handle; _backend_id = backend_id; return 0; } int unload_backend() { int dlclose_error = 0; for (auto& it : _loaded_lib_handle) { dlclose_error = dlclose(it.second); if (dlclose_error != 0) { QNN_LOG_WARN("failed to close QNN backend %d, error %s\n", it.first, dlerror()); } } _loaded_lib_handle.clear(); _lib_path_to_backend_id.clear(); _loaded_backend.clear(); return 0; } void set_qnn_raw_interface(QNN_INTERFACE_VER_TYPE& raw_interface) { _qnn_raw_interface = raw_interface; } void set_qnn_raw_system_interface(QNN_SYSTEM_INTERFACE_VER_TYPE& raw_interface) { _qnn_raw_system_interface = raw_interface; } private: static constexpr const int _required_num_providers = 1; std::string _lib_path; std::string _backend_name; std::string _model_name; // Qualcomm's dedicated prebuilt model name, keep it for further usage BackendIdType _backend_id; bool _debug_tensor = false; bool _do_node_validations = true; QnnLog_Level_t _qnn_log_level = QNN_LOG_LEVEL_DEBUG; qnn::sdk_profile_level _profile_level = qnn::sdk_profile_level::profile_detail; qnn_interface _qnn_interface; void* _system_lib_handle = nullptr; Qnn_GraphHandle_t _qnn_graph_handle = nullptr; Qnn_LogHandle_t _qnn_log_handle = nullptr; Qnn_ProfileHandle_t _qnn_profile_handle = nullptr; Qnn_DeviceHandle_t _qnn_device_handle = nullptr; Qnn_BackendHandle_t _qnn_backend_handle = nullptr; Qnn_ContextHandle_t _qnn_context_handle = nullptr; QnnSystemContext_Handle_t _qnn_system_handle = nullptr; QnnHtpDevice_PerfInfrastructure_t* _qnn_htp_perfinfra = nullptr; uint32_t _qnn_power_configid = 1; QNN_INTERFACE_VER_TYPE _qnn_raw_interface; QNN_SYSTEM_INTERFACE_VER_TYPE _qnn_raw_system_interface; std::unordered_map _qnn_mem_set; std::mutex _init_mutex; std::unordered_map _loaded_lib_handle; std::unordered_map _lib_path_to_backend_id; std::unordered_map _loaded_backend; void* _rpc_lib_handle = nullptr; std::atomic_bool _rpcmem_initialized{ false }; qnn::pfn_rpc_mem_alloc _pfn_rpc_mem_alloc; qnn::pfn_rpc_mem_free _pfn_rpc_mem_free; qnn::pfn_rpc_mem_to_fd _pfn_rpc_mem_to_fd; qnn::pfn_rpc_mem_init _pfn_rpc_mem_init; qnn::pfn_rpc_mem_deinit _pfn_rpc_mem_deinit; std::unordered_map _rpcmem_store_map; size_t _rpcmem_capacity = 512; std::string _graph_name; qnn::qcom_socinfo _soc_info = {}; }; }