257 lines
9.8 KiB
C++
257 lines
9.8 KiB
C++
#include <cuda_runtime.h>
|
|
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <cstdint>
|
|
#include <iostream>
|
|
#include <opencv2/opencv.hpp>
|
|
|
|
#include "config.h"
|
|
#include "cuda_utils.h"
|
|
#include "logging.h"
|
|
#include "model.h"
|
|
#include "postprocess.h"
|
|
#include "utils.h"
|
|
|
|
using namespace nvinfer1;
|
|
|
|
static Logger gLogger;
|
|
// const static int kOutputSize = kMaxNumOutputBbox * sizeof(Detection) / sizeof(float) + 1;
|
|
const static int kInputSize = 3 * 256 * 256;
|
|
const static int kOutputSize = 1 * 256 * 256;
|
|
|
|
bool parse_args(int argc, char** argv, std::string& wts, std::string& engine, float& gd, float& gw,
|
|
std::string& img_dir) {
|
|
if (argc != 4)
|
|
return false;
|
|
if (std::string(argv[1]) == "-s") {
|
|
wts = std::string(argv[2]);
|
|
engine = std::string(argv[3]);
|
|
} else if (std::string(argv[1]) == "-d") {
|
|
engine = std::string(argv[2]);
|
|
img_dir = std::string(argv[3]);
|
|
} else {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void prepare_infer_buffers(ICudaEngine* engine, float** gpu_input_buffer, float** gpu_output_buffer,
|
|
float** cpu_output_buffer) {
|
|
// assert(engine->getNbIOTensors() == 2);
|
|
assert(engine->getNbBindings() == 2);
|
|
|
|
// In order to bind the buffers, we need to know the names of the input and output tensors.
|
|
// Note that indices are guaranteed to be less than IEngine::getNbBindings()
|
|
const int inputIndex = engine->getBindingIndex(kInputTensorName);
|
|
const int outputIndex = engine->getBindingIndex(kOutputTensorName);
|
|
// nvinfer1::Dims outputDims = engine->getBindingDimensions(outputIndex);
|
|
assert(inputIndex == 0);
|
|
assert(outputIndex == 1);
|
|
|
|
// Create GPU in/output buffers on device
|
|
CUDA_CHECK(cudaMalloc((void**)gpu_input_buffer, kBatchSize * 3 * kInputH * kInputW * sizeof(float)));
|
|
CUDA_CHECK(cudaMalloc((void**)gpu_output_buffer, kBatchSize * 1 * kOutputSize * sizeof(float))); // 3 or 1 ??
|
|
// Create CPU output buffers on host
|
|
*cpu_output_buffer = new float[kBatchSize * kOutputSize];
|
|
}
|
|
|
|
void preprocessImg(cv::Mat& img, int newh, int neww) {
|
|
cv::cvtColor(img, img, cv::COLOR_BGR2RGB);
|
|
cv::resize(img, img, cv::Size(neww, newh));
|
|
img.convertTo(img, CV_32FC3);
|
|
// ImageNet normalize
|
|
img /= 255.0f;
|
|
img -= cv::Scalar(0.485, 0.456, 0.406);
|
|
img /= cv::Scalar(0.229, 0.224, 0.225);
|
|
}
|
|
|
|
void infer(IExecutionContext& context, cudaStream_t& stream, std::vector<void*>& gpu_buffers,
|
|
std::vector<float>& cpu_input_data, std::vector<float>& cpu_output_data, int batchsize) {
|
|
// copy input data from host (CPU) to device (GPU)
|
|
CUDA_CHECK(cudaMemcpyAsync(gpu_buffers[0], cpu_input_data.data(), cpu_input_data.size() * sizeof(float),
|
|
cudaMemcpyHostToDevice, stream));
|
|
// execute inference using context provided by engine
|
|
context.enqueue(batchsize, gpu_buffers.data(), stream, nullptr);
|
|
// copy output back from device (GPU) to host (CPU)
|
|
CUDA_CHECK(cudaMemcpyAsync(cpu_output_data.data(), gpu_buffers[1], batchsize * kOutputSize * sizeof(float),
|
|
cudaMemcpyDeviceToHost, stream));
|
|
// synchronize the stream to prevent issues (block CUDA and wait for CUDA operations to be completed)
|
|
cudaStreamSynchronize(stream);
|
|
}
|
|
|
|
void serialize_engine(unsigned int max_batchsize, float& gd, float& gw, std::string& wts_name,
|
|
std::string& engine_name) {
|
|
// Create builder
|
|
IBuilder* builder = createInferBuilder(gLogger);
|
|
IBuilderConfig* config = builder->createBuilderConfig();
|
|
|
|
// Create model to populate the network, then set the outputs and create an engine
|
|
ICudaEngine* engine = nullptr;
|
|
engine = build_efficientAD_engine(max_batchsize, builder, config, DataType::kFLOAT, gd, gw, wts_name);
|
|
assert(engine != nullptr);
|
|
|
|
// Serialize the engine
|
|
IHostMemory* serialized_engine = engine->serialize();
|
|
assert(serialized_engine != nullptr);
|
|
|
|
// Save engine to file
|
|
std::ofstream p(engine_name, std::ios::binary);
|
|
if (!p) {
|
|
std::cerr << "Could not open plan output file" << std::endl;
|
|
assert(false);
|
|
}
|
|
p.write(reinterpret_cast<const char*>(serialized_engine->data()), serialized_engine->size());
|
|
|
|
// Close everything down
|
|
engine->destroy();
|
|
config->destroy();
|
|
serialized_engine->destroy();
|
|
builder->destroy();
|
|
}
|
|
|
|
void deserialize_engine(std::string& engine_name, IRuntime** runtime, ICudaEngine** engine,
|
|
IExecutionContext** context) {
|
|
std::ifstream file(engine_name, std::ios::binary);
|
|
if (!file.good()) {
|
|
std::cerr << "read " << engine_name << " error!" << std::endl;
|
|
assert(false);
|
|
}
|
|
size_t size = 0;
|
|
file.seekg(0, file.end);
|
|
size = file.tellg();
|
|
file.seekg(0, file.beg);
|
|
char* serialized_engine = new char[size];
|
|
assert(serialized_engine);
|
|
file.read(serialized_engine, size);
|
|
file.close();
|
|
|
|
*runtime = createInferRuntime(gLogger);
|
|
assert(*runtime);
|
|
*engine = (*runtime)->deserializeCudaEngine(serialized_engine, size);
|
|
assert(*engine != nullptr);
|
|
*context = (*engine)->createExecutionContext();
|
|
assert(*context);
|
|
|
|
delete[] serialized_engine;
|
|
}
|
|
|
|
int main(int argc, char** argv) {
|
|
cudaSetDevice(kGpuId);
|
|
|
|
std::string wts_name = "";
|
|
std::string engine_name = "";
|
|
float gd = 1.0f, gw = 1.0f;
|
|
std::string img_dir;
|
|
|
|
if (!parse_args(argc, argv, wts_name, engine_name, gd, gw, img_dir)) {
|
|
std::cerr << "arguments not right!" << std::endl;
|
|
std::cerr << "./efficientad_det -s [.wts] [.engine] // serialize model to plan file" << std::endl;
|
|
std::cerr
|
|
<< "./efficientad_det -d [.engine] [../../datas/images/...] // deserialize plan file and run inference"
|
|
<< std::endl;
|
|
return -1;
|
|
}
|
|
|
|
// Create a model using the API directly and serialize it to a file
|
|
if (!wts_name.empty()) {
|
|
serialize_engine(kBatchSize, gd, gw, wts_name, engine_name);
|
|
return 0;
|
|
}
|
|
|
|
// Deserialize the engine from file
|
|
IRuntime* runtime = nullptr;
|
|
ICudaEngine* engine = nullptr;
|
|
IExecutionContext* context = nullptr;
|
|
deserialize_engine(engine_name, &runtime, &engine, &context);
|
|
|
|
// create CUDA stream for simultaneous CUDA operations
|
|
cudaStream_t stream;
|
|
CUDA_CHECK(cudaStreamCreate(&stream));
|
|
|
|
// prepare cpu and gpu buffers
|
|
void *gpu_input_buffer, *gpu_output_buffer;
|
|
CUDA_CHECK(cudaMalloc(&gpu_input_buffer, kBatchSize * 3 * kInputH * kInputW * sizeof(float)));
|
|
CUDA_CHECK(cudaMalloc(&gpu_output_buffer, kBatchSize * 1 * kOutputSize * sizeof(float))); // 3 or 1 ??
|
|
std::vector<void*> gpu_buffers = {gpu_input_buffer, gpu_output_buffer};
|
|
std::vector<float> cpu_input_data(kBatchSize * kInputSize, 0);
|
|
std::vector<float> cpu_output_data(kBatchSize * kOutputSize, 0);
|
|
|
|
// read images from directory
|
|
std::vector<std::string> file_names;
|
|
if (read_files_in_dir(img_dir.c_str(), file_names) < 0) {
|
|
std::cerr << "read_files_in_dir failed." << std::endl;
|
|
return -1;
|
|
}
|
|
|
|
std::vector<cv::Mat> originImg_batch;
|
|
for (size_t i = 0; i < file_names.size(); i += kBatchSize) {
|
|
// get a batch of images
|
|
std::vector<cv::Mat> img_batch;
|
|
std::vector<std::string> img_name_batch;
|
|
|
|
for (size_t j = i; j < i + kBatchSize && j < file_names.size(); j++) {
|
|
cv::Mat img = cv::imread(img_dir + "/" + file_names[j]);
|
|
originImg_batch.push_back(img.clone());
|
|
preprocessImg(img, kInputW, kInputH);
|
|
assert(img.cols * img.rows * 3 == 3 * 256 * 256);
|
|
for (int c = 0; c < 3; c++) {
|
|
for (int h = 0; h < img.rows; h++) {
|
|
for (int w = 0; w < img.cols; w++) {
|
|
cpu_input_data[c * img.rows * img.cols + h * img.cols + w] = img.at<cv::Vec3f>(h, w)[c];
|
|
}
|
|
}
|
|
}
|
|
img_batch.push_back(img);
|
|
img_name_batch.push_back(file_names[j]);
|
|
}
|
|
|
|
// Run inference
|
|
auto start = std::chrono::system_clock::now();
|
|
// infer(*context, stream, (void**)gpu_buffers, cpu_input_data, cpu_output_buffer, kBatchSize);
|
|
infer(*context, stream, gpu_buffers, cpu_input_data, cpu_output_data,
|
|
kBatchSize); // change to save into vec `cpu_output_data`
|
|
auto end = std::chrono::system_clock::now();
|
|
std::cout << "inference time: " << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
|
|
<< "ms" << std::endl;
|
|
|
|
// postProcess
|
|
cv::Mat img_1(256, 256, CV_8UC1);
|
|
for (int row = 0; row < 256; row++) {
|
|
for (int col = 0; col < 256; col++) {
|
|
float value = cpu_output_data[row * 256 + col];
|
|
if (value < 0) // clip(0,1)
|
|
value = 0;
|
|
else if (value > 1)
|
|
value = 1;
|
|
img_1.at<uchar>(row, col) = static_cast<uchar>(value * 255);
|
|
}
|
|
}
|
|
|
|
cv::Mat HeatMap, colorMap;
|
|
// genHeatMap(img_batch[0], img_1, HeatMap);
|
|
cv::applyColorMap(img_1, colorMap, cv::COLORMAP_JET);
|
|
cv::resize(originImg_batch[i], originImg_batch[i], cv::Size(256, 256));
|
|
cv::cvtColor(originImg_batch[i], originImg_batch[i], cv::COLOR_RGB2BGR);
|
|
cv::addWeighted(originImg_batch[i], 0.5, colorMap, 0.5, 0, HeatMap);
|
|
|
|
// Save images
|
|
for (size_t j = 0; j < img_batch.size(); j++) {
|
|
cv::imwrite("_output" + img_name_batch[j], img_1);
|
|
cv::imwrite("_heatmap" + img_name_batch[j], HeatMap);
|
|
}
|
|
}
|
|
|
|
// Release stream and buffers
|
|
cudaStreamDestroy(stream);
|
|
CUDA_CHECK(cudaFree(gpu_buffers[0]));
|
|
CUDA_CHECK(cudaFree(gpu_buffers[1]));
|
|
|
|
// Destroy the engine
|
|
context->destroy();
|
|
engine->destroy();
|
|
runtime->destroy();
|
|
|
|
return 0;
|
|
}
|