refactor: remove deps of userdma lib
This commit is contained in:
chraac
parent
a021171909
commit
38ae191c55
|
|
@ -188,14 +188,12 @@ else()
|
|||
file(GLOB common_srcs "${CMAKE_CURRENT_LIST_DIR}/common/*.cpp")
|
||||
file(GLOB device_srcs "${CMAKE_CURRENT_LIST_DIR}/device/*.cpp")
|
||||
file(GLOB device_op_srcs "${CMAKE_CURRENT_LIST_DIR}/device/op/*.cpp")
|
||||
file(GLOB dma_srcs "${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/src/*.c")
|
||||
set(skel_srcs "${CMAKE_CURRENT_BINARY_DIR}/npu_device_skel.c")
|
||||
add_library(hexagon_npu_skel_OBJS OBJECT
|
||||
${common_srcs}
|
||||
${device_srcs}
|
||||
${device_op_srcs}
|
||||
${skel_srcs}
|
||||
${dma_srcs}
|
||||
)
|
||||
|
||||
if(CMAKE_BUILD_TYPE MATCHES "Debug|Dbg")
|
||||
|
|
@ -243,8 +241,6 @@ else()
|
|||
${HEXAGON_SDK_ROOT}/libs/qprintf/inc/
|
||||
|
||||
# TODO: find a better way to include these
|
||||
${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/api/
|
||||
${HEXAGON_SDK_ROOT}/addons/compute/libs/userdma/utils_lib/inc/
|
||||
${CMAKE_CURRENT_LIST_DIR}/device/
|
||||
${CMAKE_CURRENT_LIST_DIR}/device/op/
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,228 @@
|
|||
#include "dma_transfer.hpp"
|
||||
|
||||
#include <dma_desc.h>
|
||||
#include <qurt.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace {
|
||||
|
||||
// From addons/compute/libs/userdma/utils_lib/
|
||||
|
||||
#define DM0_STATUS_MASK 0x00000003
|
||||
#define DM0_STATUS_SHIFT 0
|
||||
#define DM0_STATUS_IDLE 0
|
||||
#define DM0_STATUS_RUN 1
|
||||
#define DM0_STATUS_ERROR 2
|
||||
|
||||
#define DM0_DESC_ADDR_MASK 0xFFFFFFF0
|
||||
#define DM0_DESC_ADDR_SHIFT 4
|
||||
|
||||
#define DMA_COMPLETE 1
|
||||
#define DMA_INCOMPLETE 0
|
||||
|
||||
#define DMA_SUCCESS 0
|
||||
#define DMA_FAIL -1
|
||||
|
||||
#define DMA_DESC_TYPE_1D 0
|
||||
#define DMA_DESC_TYPE_2D 1
|
||||
|
||||
#define DESC_NEXT_MASK 0xFFFFFFFF
|
||||
#define DESC_NEXT_SHIFT 0
|
||||
|
||||
#define DESC_DSTATE_MASK 0x80000000
|
||||
#define DESC_DSTATE_SHIFT 31
|
||||
#define DESC_DSTATE_INCOMPLETE 0
|
||||
#define DESC_DSTATE_COMPLETE 1
|
||||
|
||||
#define DESC_ORDER_MASK 0x40000000
|
||||
#define DESC_ORDER_SHIFT 30
|
||||
#define DESC_ORDER_NOORDER 0
|
||||
#define DESC_ORDER_ORDER 1
|
||||
|
||||
#define DESC_BYPASSSRC_MASK 0x20000000
|
||||
#define DESC_BYPASSSRC_SHIFT 29
|
||||
#define DESC_BYPASSDST_MASK 0x10000000
|
||||
#define DESC_BYPASSDST_SHIFT 28
|
||||
#define DESC_BYPASS_OFF 0
|
||||
#define DESC_BYPASS_ON 1
|
||||
|
||||
#define DESC_DESCTYPE_MASK 0x03000000
|
||||
#define DESC_DESCTYPE_SHIFT 24
|
||||
#define DESC_DESCTYPE_1D 0
|
||||
#define DESC_DESCTYPE_2D 1
|
||||
|
||||
#define DESC_LENGTH_MASK 0x00FFFFFF
|
||||
#define DESC_LENGTH_SHIFT 0
|
||||
#define DESC_SRC_MASK 0xFFFFFFFF
|
||||
#define DESC_SRC_SHIFT 0
|
||||
#define DESC_DST_MASK 0xFFFFFFFF
|
||||
#define DESC_DST_SHIFT 0
|
||||
|
||||
#define DESC_CACHEALLOC_MASK 0x03000000
|
||||
#define DESC_CACHEALLOC_SHIFT 24
|
||||
#define DESC_CACHEALLOC_NONE 0
|
||||
#define DESC_CACHEALLOC_WRITEONLY 1
|
||||
#define DESC_CACHEALLOC_READONLY 2
|
||||
#define DESC_CACHEALLOC_READWRITE 3
|
||||
|
||||
#define DESC_ROIWIDTH_MASK 0x0000FFFF
|
||||
#define DESC_ROIWIDTH_SHIFT 0
|
||||
#define DESC_ROIHEIGHT_MASK 0xFFFF0000
|
||||
#define DESC_ROIHEIGHT_SHIFT 16
|
||||
|
||||
#define DESC_SRCSTRIDE_MASK 0x0000FFFF
|
||||
#define DESC_SRCSTRIDE_SHIFT 0
|
||||
#define DESC_DSTSTRIDE_MASK 0xFFFF0000
|
||||
#define DESC_DSTSTRIDE_SHIFT 16
|
||||
|
||||
#define DESC_SRCWIDTHOFFSET_MASK 0x0000FFFF
|
||||
#define DESC_SRCWIDTHOFFSET_SHIFT 0
|
||||
#define DESC_DSTWIDTHOFFSET_MASK 0xFFFF0000
|
||||
#define DESC_DSTWIDTHOFFSET_SHIFT 16
|
||||
|
||||
/**************************/
|
||||
/* 1D (linear) descriptor */
|
||||
/**************************/
|
||||
typedef struct _dma_desc_1d_t {
|
||||
uint32_t next;
|
||||
uint32_t dstate_order_bypass_desctype_length;
|
||||
uint32_t src;
|
||||
uint32_t dst;
|
||||
} dma_desc_1d_t;
|
||||
|
||||
static_assert(sizeof(dma_desc_1d_t) == hexagon::dma::kDmaDescSize1D, "kDmaDescSize1D size incorrect");
|
||||
|
||||
/***********************/
|
||||
/* 2D (box) descriptor */
|
||||
/***********************/
|
||||
typedef struct _dma_desc_2d_t {
|
||||
uint32_t next;
|
||||
uint32_t dstate_order_bypass_desctype_length;
|
||||
uint32_t src;
|
||||
uint32_t dst;
|
||||
uint32_t allocation;
|
||||
uint32_t roiheight_roiwidth;
|
||||
uint32_t dststride_srcstride;
|
||||
uint32_t dstwidthoffset_srcwidthoffset;
|
||||
} dma_desc_2d_t;
|
||||
|
||||
static_assert(sizeof(dma_desc_2d_t) == hexagon::dma::kDmaDescSize2D, "kDmaDescSize2D size incorrect");
|
||||
|
||||
inline void dmstart(void * next) {
|
||||
asm volatile(" release(%0):at" : : "r"(next));
|
||||
asm volatile(" dmstart(%0)" : : "r"(next));
|
||||
}
|
||||
|
||||
inline void dmlink(void * cur, void * next) {
|
||||
asm volatile(" release(%0):at" : : "r"(next));
|
||||
asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
|
||||
}
|
||||
|
||||
inline unsigned int dmpoll(void) {
|
||||
unsigned int ret = 0;
|
||||
asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline unsigned int dmwait(void) {
|
||||
unsigned int ret = 0;
|
||||
asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
|
||||
return ret;
|
||||
}
|
||||
|
||||
inline void dma_desc_set_next(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->next) &= ~DESC_NEXT_MASK;
|
||||
(((dma_desc_1d_t *) d)->next) |= ((v << DESC_NEXT_SHIFT) & DESC_NEXT_MASK);
|
||||
}
|
||||
|
||||
inline uint32_t dma_desc_get_dstate(void * d) {
|
||||
return (((((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) & DESC_DSTATE_MASK) >> DESC_DSTATE_SHIFT);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_dstate(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DSTATE_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DSTATE_SHIFT) & DESC_DSTATE_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_desctype(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_DESCTYPE_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_DESCTYPE_SHIFT) & DESC_DESCTYPE_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_order(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_ORDER_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_ORDER_SHIFT) & DESC_ORDER_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_bypasssrc(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSSRC_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSSRC_SHIFT) & DESC_BYPASSSRC_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_bypassdst(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_BYPASSDST_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_BYPASSDST_SHIFT) & DESC_BYPASSDST_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_length(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) &= ~DESC_LENGTH_MASK;
|
||||
(((dma_desc_1d_t *) d)->dstate_order_bypass_desctype_length) |= ((v << DESC_LENGTH_SHIFT) & DESC_LENGTH_MASK);
|
||||
}
|
||||
|
||||
inline uint32_t dma_desc_get_src(void * d) {
|
||||
return (((((dma_desc_1d_t *) d)->src) & DESC_SRC_MASK) >> DESC_SRC_SHIFT);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_src(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->src) &= ~DESC_SRC_MASK;
|
||||
(((dma_desc_1d_t *) d)->src) |= ((v << DESC_SRC_SHIFT) & DESC_SRC_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_dst(void * d, uint32_t v) {
|
||||
(((dma_desc_1d_t *) d)->dst) &= ~DESC_DST_MASK;
|
||||
(((dma_desc_1d_t *) d)->dst) |= ((v << DESC_DST_SHIFT) & DESC_DST_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_roiwidth(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIWIDTH_MASK;
|
||||
(((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIWIDTH_SHIFT) & DESC_ROIWIDTH_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_roiheight(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->roiheight_roiwidth) &= ~DESC_ROIHEIGHT_MASK;
|
||||
(((dma_desc_2d_t *) d)->roiheight_roiwidth) |= ((v << DESC_ROIHEIGHT_SHIFT) & DESC_ROIHEIGHT_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_srcstride(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_SRCSTRIDE_MASK;
|
||||
(((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_SRCSTRIDE_SHIFT) & DESC_SRCSTRIDE_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_dststride(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->dststride_srcstride) &= ~DESC_DSTSTRIDE_MASK;
|
||||
(((dma_desc_2d_t *) d)->dststride_srcstride) |= ((v << DESC_DSTSTRIDE_SHIFT) & DESC_DSTSTRIDE_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_srcwidthoffset(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_SRCWIDTHOFFSET_MASK;
|
||||
(((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |=
|
||||
((v << DESC_SRCWIDTHOFFSET_SHIFT) & DESC_SRCWIDTHOFFSET_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_dstwidthoffset(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) &= ~DESC_DSTWIDTHOFFSET_MASK;
|
||||
(((dma_desc_2d_t *) d)->dstwidthoffset_srcwidthoffset) |=
|
||||
((v << DESC_DSTWIDTHOFFSET_SHIFT) & DESC_DSTWIDTHOFFSET_MASK);
|
||||
}
|
||||
|
||||
inline void dma_desc_set_cachealloc(void * d, uint32_t v) {
|
||||
(((dma_desc_2d_t *) d)->allocation) &= ~DESC_CACHEALLOC_MASK;
|
||||
(((dma_desc_2d_t *) d)->allocation) |= ((v << DESC_CACHEALLOC_SHIFT) & DESC_CACHEALLOC_MASK);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace hexagon::dma {
|
||||
|
||||
dma_transfer::dma_transfer() {
|
||||
|
|
@ -162,23 +379,58 @@ bool dma_transfer::submit2d(const uint8_t * src,
|
|||
}
|
||||
|
||||
void dma_transfer::wait() {
|
||||
auto ret = dma_wait_for_idle();
|
||||
if (ret != DMA_SUCCESS) {
|
||||
DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle: %d\n", ret);
|
||||
uint32_t dm0_status = dmwait() & DM0_STATUS_MASK;
|
||||
if (dm0_status != DM0_STATUS_IDLE) {
|
||||
DEVICE_LOG_ERROR("dma_transfer: failed to wait for DMA idle, dm0_status: %d\n", (int) dm0_status);
|
||||
}
|
||||
}
|
||||
|
||||
bool dma_transfer::is_desc_done(uint8_t * desc) {
|
||||
return !dma_desc_get_src(desc) || dma_desc_is_done(desc) == DMA_COMPLETE;
|
||||
if (!dma_desc_get_src(desc)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (dma_desc_get_dstate(desc) == DESC_DSTATE_COMPLETE) {
|
||||
return true;
|
||||
}
|
||||
|
||||
dmpoll();
|
||||
return false;
|
||||
}
|
||||
|
||||
bool dma_transfer::submit_impl(void ** desc_batch, int batch_len) {
|
||||
bool dma_transfer::submit_impl(void ** desc_batch, size_t batch_len) {
|
||||
_dma_desc_mutex.lock();
|
||||
const bool succ = dma_desc_submit(desc_batch, batch_len) == DMA_SUCCESS;
|
||||
for (size_t i = 0; i < batch_len - 1; i++) {
|
||||
dma_desc_set_next(desc_batch[i], (uint32_t) desc_batch[i + 1]);
|
||||
}
|
||||
|
||||
dma_desc_set_next(desc_batch[batch_len - 1], (uint32_t) nullptr);
|
||||
uint32_t dm0_status = dmpoll() & DM0_STATUS_MASK;
|
||||
if (dm0_status == DM0_STATUS_IDLE) {
|
||||
dmstart(desc_batch[0]);
|
||||
} else if (dm0_status == DM0_STATUS_RUN) {
|
||||
if (_dma_last_desc == nullptr) {
|
||||
_dma_desc_mutex.unlock();
|
||||
DEVICE_LOG_ERROR("dma_transfer: last descriptor not found for linking. Submission failed\n");
|
||||
return false;
|
||||
} else {
|
||||
dmlink(_dma_last_desc, desc_batch[0]);
|
||||
}
|
||||
} else {
|
||||
_dma_desc_mutex.unlock();
|
||||
DEVICE_LOG_ERROR("dma_transfer: DMA not idle or running. Submission failed\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
dmpoll();
|
||||
|
||||
_dma_last_desc = (void *) desc_batch[batch_len - 1];
|
||||
|
||||
_dma_desc_mutex.unlock();
|
||||
return succ;
|
||||
return true;
|
||||
}
|
||||
|
||||
qurt_mutex dma_transfer::_dma_desc_mutex;
|
||||
void * dma_transfer::_dma_last_desc = nullptr;
|
||||
|
||||
} // namespace hexagon::dma
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
|
||||
#include "util.hpp"
|
||||
|
||||
#include <dma_utils.h>
|
||||
|
||||
namespace hexagon::dma {
|
||||
|
||||
constexpr const size_t kDmaDescSize1D = 16;
|
||||
constexpr const size_t kDmaDescSize2D = 32;
|
||||
|
||||
class dma_transfer {
|
||||
public:
|
||||
dma_transfer();
|
||||
|
|
@ -32,12 +33,14 @@ class dma_transfer {
|
|||
private:
|
||||
static bool is_desc_done(uint8_t * desc); // TODO: should we use void * here?
|
||||
static qurt_mutex _dma_desc_mutex;
|
||||
static void * _dma_last_desc;
|
||||
|
||||
bool submit_impl(void ** desc_batch, int batch_len);
|
||||
// TODO: can we avoid the void ** here?
|
||||
bool submit_impl(void ** desc_batch, size_t batch_len);
|
||||
|
||||
alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc0[DMA_DESC_SIZE_1D] = {};
|
||||
alignas(DMA_DESC_SIZE_1D) uint8_t _dma_1d_desc1[DMA_DESC_SIZE_1D] = {};
|
||||
alignas(DMA_DESC_SIZE_2D) uint8_t _dma_2d_desc0[DMA_DESC_SIZE_2D] = {};
|
||||
alignas(kDmaDescSize1D) uint8_t _dma_1d_desc0[kDmaDescSize1D] = {};
|
||||
alignas(kDmaDescSize1D) uint8_t _dma_1d_desc1[kDmaDescSize1D] = {};
|
||||
alignas(kDmaDescSize2D) uint8_t _dma_2d_desc0[kDmaDescSize2D] = {};
|
||||
|
||||
DISABLE_COPY_AND_MOVE(dma_transfer);
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue