diff --git a/ggml/src/ggml-qnn/npu/CMakeLists.txt b/ggml/src/ggml-qnn/npu/CMakeLists.txt index 838cfdd054..a5b6d18e34 100644 --- a/ggml/src/ggml-qnn/npu/CMakeLists.txt +++ b/ggml/src/ggml-qnn/npu/CMakeLists.txt @@ -188,14 +188,12 @@ else() file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp") file(GLOB device_srcs "${CMAKE_CURRENT_LIST_DIR}/device/*.cpp") file(GLOB device_op_srcs "${CMAKE_CURRENT_LIST_DIR}/device/op/*.cpp") - file(GLOB dma_srcs "${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/src/*.c") set(skel_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_skel.c") add_library(hexagon_npu_skel_OBJS OBJECT ${common_srcs} ${device_srcs} ${device_op_srcs} ${skel_srcs} - ${dma_srcs} ) if(CMAKE_BUILD_TYPE MATCHES "Debug|Dbg") @@ -243,8 +241,6 @@ else() ${HEXAGON_SDK_ROOT}/libs/qprintf/inc/ # TODO: find a better way to include these - ${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/api/ - ${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/inc/ ${CMAKE_CURRENT_LIST_DIR}/device/ ${CMAKE_CURRENT_LIST_DIR}/device/op/ ) diff --git a/ggml/src/ggml-qnn/npu/device/dma_transfer.cpp b/ggml/src/ggml-qnn/npu/device/dma_transfer.cpp index 27bef6c719..3a8cd0a4a3 100644 --- a/ggml/src/ggml-qnn/npu/device/dma_transfer.cpp +++ b/ggml/src/ggml-qnn/npu/device/dma_transfer.cpp @@ -1,11 +1,228 @@ #include "dma_transfer.hpp" -#include #include #include #include +namespace { + +// From addons/compute/libs/userdma/utils_lib/ + +#define DM0_STATUS_MASK 0x00000003 +#define DM0_STATUS_SHIFT 0 +#define DM0_STATUS_IDLE 0 +#define DM0_STATUS_RUN 1 +#define DM0_STATUS_ERROR 2 + +#define DM0_DESC_ADDR_MASK 0xFFFFFFF0 +#define DM0_DESC_ADDR_SHIFT 4 + +#define DMA_COMPLETE 1 +#define DMA_INCOMPLETE 0 + +#define DMA_SUCCESS 0 +#define DMA_FAIL -1 + +#define DMA_DESC_TYPE_1D 0 +#define DMA_DESC_TYPE_2D 1 + +#define DESC_NEXT_MASK 0xFFFFFFFF +#define DESC_NEXT_SHIFT 0 + +#define DESC_DSTATE_MASK 0x80000000 +#define DESC_DSTATE_SHIFT 31 +#define DESC_DSTATE_INCOMPLETE 0 +#define DESC_DSTATE_COMPLETE 1 + +#define DESC_ORDER_MASK 0x40000000 +#define DESC_ORDER_SHIFT 30 +#define DESC_ORDER_NOORDER 0 +#define DESC_ORDER_ORDER 1 + +#define DESC_BYPASSSRC_MASK 0x20000000 +#define DESC_BYPASSSRC_SHIFT 29 +#define DESC_BYPASSDST_MASK 0x10000000 +#define DESC_BYPASSDST_SHIFT 28 +#define DESC_BYPASS_OFF 0 +#define DESC_BYPASS_ON 1 + +#define DESC_DESCTYPE_MASK 0x03000000 +#define DESC_DESCTYPE_SHIFT 24 +#define DESC_DESCTYPE_1D 0 +#define DESC_DESCTYPE_2D 1 + +#define DESC_LENGTH_MASK 0x00FFFFFF +#define DESC_LENGTH_SHIFT 0 +#define DESC_SRC_MASK 0xFFFFFFFF +#define DESC_SRC_SHIFT 0 +#define DESC_DST_MASK 0xFFFFFFFF +#define DESC_DST_SHIFT 0 + +#define DESC_CACHEALLOC_MASK 0x03000000 +#define DESC_CACHEALLOC_SHIFT 24 +#define DESC_CACHEALLOC_NONE 0 +#define DESC_CACHEALLOC_WRITEONLY 1 +#define DESC_CACHEALLOC_READONLY 2 +#define DESC_CACHEALLOC_READWRITE 3 + +#define DESC_ROIWIDTH_MASK 0x0000FFFF +#define DESC_ROIWIDTH_SHIFT 0 +#define DESC_ROIHEIGHT_MASK 0xFFFF0000 +#define DESC_ROIHEIGHT_SHIFT 16 + +#define DESC_SRCSTRIDE_MASK 0x0000FFFF +#define DESC_SRCSTRIDE_SHIFT 0 +#define DESC_DSTSTRIDE_MASK 0xFFFF0000 +#define DESC_DSTSTRIDE_SHIFT 16 + +#define DESC_SRCWIDTHOFFSET_MASK 0x0000FFFF +#define DESC_SRCWIDTHOFFSET_SHIFT 0 +#define DESC_DSTWIDTHOFFSET_MASK 0xFFFF0000 +#define DESC_DSTWIDTHOFFSET_SHIFT 16 + +/**************************/ +/* 1D (linear) descriptor */ +/**************************/ +typedef struct _dma_desc_1d_t { + uint32_t next; + uint32_t dstate_order_bypass_desctype_length; + uint32_t src; + uint32_t dst; +} dma_desc_1d_t; + +static_assert(sizeof(dma_desc_1d_t) == hexagon::dma::kDmaDescSize1D, "kDmaDescSize1D size incorrect"); + +/***********************/ +/* 2D (box) descriptor */ +/***********************/ +typedef struct _dma_desc_2d_t { + uint32_t next; + uint32_t dstate_order_bypass_desctype_length; + uint32_t src; + uint32_t dst; + uint32_t allocation; + uint32_t roiheight_roiwidth; + uint32_t dststride_srcstride; + uint32_t dstwidthoffset_srcwidthoffset; +} dma_desc_2d_t; + +static_assert(sizeof(dma_desc_2d_t) == hexagon::dma::kDmaDescSize2D, "kDmaDescSize2D size incorrect"); + +inline void dmstart(void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmstart(%0)" : : "r"(next)); +} + +inline void dmlink(void * cur, void * next) { + asm volatile(" release(%0):at" : : "r"(next)); + asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next)); +} + +inline unsigned int dmpoll(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory"); + return ret; +} + +inline unsigned int dmwait(void) { + unsigned int ret = 0; + asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory"); + return ret; +} + +inline void dma_desc_set_next(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->next) &= ~DESC_NEXT_MASK; + (((dma_desc_1d_t *) d)->next) |= ((v << DESC_NEXT_SHIFT) & DESC_NEXT_MASK); +} + +inline uint32_t dma_desc_get_dstate(void * d) { + return (((((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) & DESC_DSTATE_MASK) >> DESC_DSTATE_SHIFT); +} + +inline void dma_desc_set_dstate(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DSTATE_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DSTATE_SHIFT) & DESC_DSTATE_MASK); +} + +inline void dma_desc_set_desctype(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DESCTYPE_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DESCTYPE_SHIFT) & DESC_DESCTYPE_MASK); +} + +inline void dma_desc_set_order(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_ORDER_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_ORDER_SHIFT) & DESC_ORDER_MASK); +} + +inline void dma_desc_set_bypasssrc(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSSRC_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSSRC_SHIFT) & DESC_BYPASSSRC_MASK); +} + +inline void dma_desc_set_bypassdst(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSDST_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSDST_SHIFT) & DESC_BYPASSDST_MASK); +} + +inline void dma_desc_set_length(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_LENGTH_MASK; + (((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_LENGTH_SHIFT) & DESC_LENGTH_MASK); +} + +inline uint32_t dma_desc_get_src(void * d) { + return (((((dma_desc_1d_t *) d)->src) & DESC_SRC_MASK) >> DESC_SRC_SHIFT); +} + +inline void dma_desc_set_src(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->src) &= ~DESC_SRC_MASK; + (((dma_desc_1d_t *) d)->src) |= ((v << DESC_SRC_SHIFT) & DESC_SRC_MASK); +} + +inline void dma_desc_set_dst(void * d, uint32_t v) { + (((dma_desc_1d_t *) d)->dst) &= ~DESC_DST_MASK; + (((dma_desc_1d_t *) d)->dst) |= ((v << DESC_DST_SHIFT) & DESC_DST_MASK); +} + +inline void dma_desc_set_roiwidth(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIWIDTH_MASK; + (((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIWIDTH_SHIFT) & DESC_ROIWIDTH_MASK); +} + +inline void dma_desc_set_roiheight(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIHEIGHT_MASK; + (((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIHEIGHT_SHIFT) & DESC_ROIHEIGHT_MASK); +} + +inline void dma_desc_set_srcstride(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_SRCSTRIDE_MASK; + (((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_SRCSTRIDE_SHIFT) & DESC_SRCSTRIDE_MASK); +} + +inline void dma_desc_set_dststride(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_DSTSTRIDE_MASK; + (((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_DSTSTRIDE_SHIFT) & DESC_DSTSTRIDE_MASK); +} + +inline void dma_desc_set_srcwidthoffset(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_SRCWIDTHOFFSET_MASK; + (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |= + ((v << DESC_SRCWIDTHOFFSET_SHIFT) & DESC_SRCWIDTHOFFSET_MASK); +} + +inline void dma_desc_set_dstwidthoffset(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_DSTWIDTHOFFSET_MASK; + (((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |= + ((v << DESC_DSTWIDTHOFFSET_SHIFT) & DESC_DSTWIDTHOFFSET_MASK); +} + +inline void dma_desc_set_cachealloc(void * d, uint32_t v) { + (((dma_desc_2d_t *) d)->allocation) &= ~DESC_CACHEALLOC_MASK; + (((dma_desc_2d_t *) d)->allocation) |= ((v << DESC_CACHEALLOC_SHIFT) & DESC_CACHEALLOC_MASK); +} + +} // namespace + namespace hexagon::dma { dma_transfer::dma_transfer() { @@ -162,23 +379,58 @@ bool dma_transfer::submit2d(const uint8_t * src, } void dma_transfer::wait() { - auto ret = dma_wait_for_idle(); - if (ret != DMA_SUCCESS) { - DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle: %d\n", ret); + uint32_t dm0_status = dmwait() & DM0_STATUS_MASK; + if (dm0_status != DM0_STATUS_IDLE) { + DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle, dm0_status: %d\n", (int) dm0_status); } } bool dma_transfer::is_desc_done(uint8_t * desc) { - return !dma_desc_get_src(desc) || dma_desc_is_done(desc) == DMA_COMPLETE; + if (!dma_desc_get_src(desc)) { + return true; + } + + if (dma_desc_get_dstate(desc) == DESC_DSTATE_COMPLETE) { + return true; + } + + dmpoll(); + return false; } -bool dma_transfer::submit_impl(void ** desc_batch, int batch_len) { +bool dma_transfer::submit_impl(void ** desc_batch, size_t batch_len) { _dma_desc_mutex.lock(); - const bool succ = dma_desc_submit(desc_batch, batch_len) == DMA_SUCCESS; + for (size_t i = 0; i < batch_len - 1; i++) { + dma_desc_set_next(desc_batch[i], (uint32_t) desc_batch[i + 1]); + } + + dma_desc_set_next(desc_batch[batch_len - 1], (uint32_t) nullptr); + uint32_t dm0_status = dmpoll() & DM0_STATUS_MASK; + if (dm0_status == DM0_STATUS_IDLE) { + dmstart(desc_batch[0]); + } else if (dm0_status == DM0_STATUS_RUN) { + if (_dma_last_desc == nullptr) { + _dma_desc_mutex.unlock(); + DEVICE_LOG_ERROR("dma_transfer: last descriptor not found for linking. Submission failed\n"); + return false; + } else { + dmlink(_dma_last_desc, desc_batch[0]); + } + } else { + _dma_desc_mutex.unlock(); + DEVICE_LOG_ERROR("dma_transfer: DMA not idle or running. Submission failed\n"); + return false; + } + + dmpoll(); + + _dma_last_desc = (void *) desc_batch[batch_len - 1]; + _dma_desc_mutex.unlock(); - return succ; + return true; } qurt_mutex dma_transfer::_dma_desc_mutex; +void * dma_transfer::_dma_last_desc = nullptr; } // namespace hexagon::dma diff --git a/ggml/src/ggml-qnn/npu/device/dma_transfer.hpp b/ggml/src/ggml-qnn/npu/device/dma_transfer.hpp index 5c14795add..86a152538f 100644 --- a/ggml/src/ggml-qnn/npu/device/dma_transfer.hpp +++ b/ggml/src/ggml-qnn/npu/device/dma_transfer.hpp @@ -2,10 +2,11 @@ #include "util.hpp" -#include - namespace hexagon::dma { +constexpr const size_t kDmaDescSize1D = 16; +constexpr const size_t kDmaDescSize2D = 32; + class dma_transfer { public: dma_transfer(); @@ -32,12 +33,14 @@ class dma_transfer { private: static bool is_desc_done(uint8_t * desc); // TODO: should we use void * here? static qurt_mutex _dma_desc_mutex; + static void * _dma_last_desc; - bool submit_impl(void ** desc_batch, int batch_len); + // TODO: can we avoid the void ** here? + bool submit_impl(void ** desc_batch, size_t batch_len); - alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc0[DMA_DESC_SIZE_1D] = {}; - alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc1[DMA_DESC_SIZE_1D] = {}; - alignas(DMA_DESC_SIZE_2D) uint8_t _dma_2d_desc0[DMA_DESC_SIZE_2D] = {}; + alignas(kDmaDescSize1D) uint8_t _dma_1d_desc0[kDmaDescSize1D] = {}; + alignas(kDmaDescSize1D) uint8_t _dma_1d_desc1[kDmaDescSize1D] = {}; + alignas(kDmaDescSize2D) uint8_t _dma_2d_desc0[kDmaDescSize2D] = {}; DISABLE_COPY_AND_MOVE(dma_transfer); };