#include "utils/dma_alloc.h" #include #include #include #include #include #include #include #if defined(__linux__) #include #include #include #include // dma_heap ioctl definitions (linux/dma-heap.h may not be available) struct dma_heap_allocation_data { uint64_t len; uint32_t fd; uint32_t fd_flags; uint64_t heap_flags; }; #define DMA_HEAP_IOC_MAGIC 'H' #define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOC_MAGIC, 0x0, struct dma_heap_allocation_data) // DMA-BUF sync ioctl struct dma_buf_sync { uint64_t flags; }; #define DMA_BUF_SYNC_READ (1 << 0) #define DMA_BUF_SYNC_WRITE (2 << 0) #define DMA_BUF_SYNC_START (0 << 2) #define DMA_BUF_SYNC_END (1 << 2) #define DMA_BUF_BASE 'b' #define DMA_BUF_IOCTL_SYNC _IOW(DMA_BUF_BASE, 0, struct dma_buf_sync) #endif namespace rk3588 { namespace { struct DmaBufferPool { std::mutex mu; std::unordered_map> free_by_size; size_t total_bytes = 0; size_t total_buffers = 0; bool inited = false; bool enabled = true; size_t max_bytes = 64ULL * 1024ULL * 1024ULL; size_t max_buffers = 16; void InitOnce() { if (inited) return; inited = true; if (const char* dis = std::getenv("RK3588_DMA_POOL_DISABLE")) { if (std::strcmp(dis, "1") == 0 || std::strcmp(dis, "true") == 0) enabled = false; } if (const char* mb = std::getenv("RK3588_DMA_POOL_MAX_BYTES")) { const long long v = std::atoll(mb); if (v > 0) max_bytes = static_cast(v); } if (const char* mc = std::getenv("RK3588_DMA_POOL_MAX_BUFFERS")) { const long long v = std::atoll(mc); if (v > 0) max_buffers = static_cast(v); } } DmaBuffer* Take(size_t size) { std::lock_guard lock(mu); InitOnce(); if (!enabled) return nullptr; auto it = free_by_size.find(size); if (it == free_by_size.end() || it->second.empty()) return nullptr; DmaBuffer* b = it->second.back(); it->second.pop_back(); total_bytes -= size; total_buffers -= 1; return b; } void Put(DmaBuffer* b) { if (!b) return; std::lock_guard lock(mu); InitOnce(); if (!enabled) { delete b; return; } const size_t sz = b->size; if (sz == 0) { delete b; return; } if (total_buffers + 1 > max_buffers || total_bytes + sz > max_bytes) { delete b; return; } free_by_size[sz].push_back(b); total_bytes += sz; total_buffers += 1; } }; static DmaBufferPool& Pool() { static DmaBufferPool* p = new DmaBufferPool(); return *p; } } // namespace DmaBuffer::~DmaBuffer() { #if defined(__linux__) if (ptr && size > 0) { munmap(ptr, size); ptr = nullptr; } if (fd >= 0) { close(fd); fd = -1; } #endif size = 0; } DmaBuffer::DmaBuffer(DmaBuffer&& other) noexcept : fd(other.fd), ptr(other.ptr), size(other.size) { other.fd = -1; other.ptr = nullptr; other.size = 0; } DmaBuffer& DmaBuffer::operator=(DmaBuffer&& other) noexcept { if (this != &other) { #if defined(__linux__) if (ptr && size > 0) { munmap(ptr, size); } if (fd >= 0) { close(fd); } #endif fd = other.fd; ptr = other.ptr; size = other.size; other.fd = -1; other.ptr = nullptr; other.size = 0; } return *this; } DmaBufferPtr DmaAlloc(size_t alloc_size) { #if defined(__linux__) if (alloc_size == 0) return nullptr; if (auto* reused = Pool().Take(alloc_size)) { return DmaBufferPtr(reused, [](DmaBuffer* b) { Pool().Put(b); }); } // Try different dma_heap devices. // Notes: // - Some RGA drivers cannot map buffers whose physical address is >4GB; prefer DMA32 heaps. // - CMA heaps are also <4GB friendly but can be small/fragmented under multi-stream load. const char* heap_paths[] = { // Prefer DMA32 heaps to keep allocations under 4GB. // This avoids RGA drivers that cannot map >4GB physical addresses. "/dev/dma_heap/system-dma32", "/dev/dma_heap/system-uncached-dma32", // CMA is also <4GB friendly, but can be small/fragmented under multi-stream load. "/dev/dma_heap/cma", "/dev/dma_heap/cma-uncached", "/dev/dma_heap/linux,cma", // Generic system heaps (may allocate >4GB on high-mem systems). "/dev/dma_heap/system-uncached", "/dev/dma_heap/system", // Last resort. "/dev/dma_heap/reserved", nullptr }; static bool first_success = true; int dma_fd = -1; const char* used_heap = nullptr; int last_errno = 0; for (int i = 0; heap_paths[i] != nullptr; ++i) { int heap_fd = open(heap_paths[i], O_RDWR | O_CLOEXEC); if (heap_fd < 0) { last_errno = errno; continue; } dma_heap_allocation_data alloc_data{}; alloc_data.len = alloc_size; alloc_data.fd_flags = O_RDWR | O_CLOEXEC; alloc_data.heap_flags = 0; if (ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &alloc_data) == 0) { dma_fd = static_cast(alloc_data.fd); used_heap = heap_paths[i]; close(heap_fd); break; } last_errno = errno; close(heap_fd); continue; } if (dma_fd < 0) { std::cerr << "[DmaAlloc] DMA_HEAP_IOCTL_ALLOC failed on all heaps: " << strerror(last_errno) << "\n"; std::cerr << "[DmaAlloc] tried: "; for (int i = 0; heap_paths[i] != nullptr; ++i) { std::cerr << heap_paths[i] << " "; } std::cerr << "\n"; return nullptr; } if (first_success) { std::cout << "[DmaAlloc] using heap: " << used_heap << "\n"; first_success = false; } void* ptr = mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_SHARED, dma_fd, 0); if (ptr == MAP_FAILED) { std::cerr << "[DmaAlloc] mmap failed: " << strerror(errno) << "\n"; close(dma_fd); return nullptr; } auto* buf = new DmaBuffer(); buf->fd = dma_fd; buf->ptr = ptr; buf->size = alloc_size; return DmaBufferPtr(buf, [](DmaBuffer* b) { Pool().Put(b); }); #else (void)alloc_size; std::cerr << "[DmaAlloc] not supported on this platform\n"; return nullptr; #endif } void DmaSyncStart(DmaBuffer* buf) { #if defined(__linux__) if (!buf || buf->fd < 0) return; dma_buf_sync sync{}; sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE; ioctl(buf->fd, DMA_BUF_IOCTL_SYNC, &sync); #else (void)buf; #endif } void DmaSyncEnd(DmaBuffer* buf) { #if defined(__linux__) if (!buf || buf->fd < 0) return; dma_buf_sync sync{}; sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE; ioctl(buf->fd, DMA_BUF_IOCTL_SYNC, &sync); #else (void)buf; #endif } void DmaSyncStartFd(int fd) { #if defined(__linux__) if (fd < 0) return; dma_buf_sync sync{}; sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE; ioctl(fd, DMA_BUF_IOCTL_SYNC, &sync); #else (void)fd; #endif } void DmaSyncEndFd(int fd) { #if defined(__linux__) if (fd < 0) return; dma_buf_sync sync{}; sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ | DMA_BUF_SYNC_WRITE; ioctl(fd, DMA_BUF_IOCTL_SYNC, &sync); #else (void)fd; #endif } } // namespace rk3588