mobilenetv3 to trt7
This commit is contained in:
wang-xinyu
parent
0e23a96d2b
commit
2aa8c18b77
@ -13,16 +13,17 @@ find_package(CUDA REQUIRED)
|
||||
set(CUDA_NVCC_PLAGS ${CUDA_NVCC_PLAGS};-std=c++11;-g;-G;-gencode;arch=compute_30;code=sm_30;)
|
||||
|
||||
include_directories(${PROJECT_SOURCE_DIR}/include)
|
||||
include_directories(/usr/local/cuda/targets/aarch64-linux/include)
|
||||
link_directories(/usr/local/cuda/targets/aarch64-linux/lib)
|
||||
cuda_add_library(h_sigmoid ${PROJECT_SOURCE_DIR}/h_sigmoid.cu)
|
||||
#cuda_add_library(leaky ${PROJECT_SOURCE_DIR}/leaky.cu)
|
||||
# include and link dirs of cuda and tensorrt, you need adapt them if yours are different
|
||||
# cuda
|
||||
include_directories(/usr/local/cuda/include)
|
||||
link_directories(/usr/local/cuda/lib64)
|
||||
# tensorrt
|
||||
include_directories(/usr/include/x86_64-linux-gnu/)
|
||||
link_directories(/usr/lib/x86_64-linux-gnu/)
|
||||
|
||||
add_executable(mobilenetv3 ${PROJECT_SOURCE_DIR}/h_sigmoidplugin.cpp ${PROJECT_SOURCE_DIR}/mobilenet_v3.cpp)
|
||||
add_executable(mobilenetv3 ${PROJECT_SOURCE_DIR}/mobilenet_v3.cpp)
|
||||
target_link_libraries(mobilenetv3 nvinfer)
|
||||
target_link_libraries(mobilenetv3 cudart)
|
||||
#target_link_libraries(mobilenetv3 leaky)
|
||||
target_link_libraries(mobilenetv3 h_sigmoid)
|
||||
|
||||
add_definitions(-O2 -pthread)
|
||||
|
||||
|
||||
@ -1,356 +0,0 @@
|
||||
#ifndef _TRT_COMMON_H_
|
||||
#define _TRT_COMMON_H_
|
||||
#include "NvInfer.h"
|
||||
#include "NvOnnxConfig.h"
|
||||
#include "NvOnnxParser.h"
|
||||
#include <cuda_runtime_api.h>
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <cmath>
|
||||
|
||||
using namespace std;
|
||||
|
||||
#define CHECK(status) \
|
||||
do \
|
||||
{ \
|
||||
auto ret = (status); \
|
||||
if (ret != 0) \
|
||||
{ \
|
||||
std::cout << "Cuda failure: " << ret; \
|
||||
abort(); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
constexpr long double operator"" _GB(long double val) { return val * (1 << 30); }
|
||||
constexpr long double operator"" _MB(long double val) { return val * (1 << 20); }
|
||||
constexpr long double operator"" _KB(long double val) { return val * (1 << 10); }
|
||||
|
||||
// These is necessary if we want to be able to write 1_GB instead of 1.0_GB.
|
||||
// Since the return type is signed, -1_GB will work as expected.
|
||||
constexpr long long int operator"" _GB(long long unsigned int val) { return val * (1 << 30); }
|
||||
constexpr long long int operator"" _MB(long long unsigned int val) { return val * (1 << 20); }
|
||||
constexpr long long int operator"" _KB(long long unsigned int val) { return val * (1 << 10); }
|
||||
|
||||
// Logger for TensorRT info/warning/errors
|
||||
class Logger : public nvinfer1::ILogger
|
||||
{
|
||||
public:
|
||||
|
||||
Logger(): Logger(Severity::kWARNING) {}
|
||||
|
||||
Logger(Severity severity): reportableSeverity(severity) {}
|
||||
|
||||
void log(Severity severity, const char* msg) override
|
||||
{
|
||||
// suppress messages with severity enum value greater than the reportable
|
||||
if (severity > reportableSeverity) return;
|
||||
|
||||
switch (severity)
|
||||
{
|
||||
case Severity::kINTERNAL_ERROR: std::cerr << "INTERNAL_ERROR: "; break;
|
||||
case Severity::kERROR: std::cerr << "ERROR: "; break;
|
||||
case Severity::kWARNING: std::cerr << "WARNING: "; break;
|
||||
case Severity::kINFO: std::cerr << "INFO: "; break;
|
||||
default: std::cerr << "UNKNOWN: "; break;
|
||||
}
|
||||
std::cerr << msg << std::endl;
|
||||
}
|
||||
|
||||
Severity reportableSeverity{Severity::kWARNING};
|
||||
};
|
||||
|
||||
// Locate path to file, given its filename or filepath suffix and possible dirs it might lie in
|
||||
// Function will also walk back MAX_DEPTH dirs from CWD to check for such a file path
|
||||
inline std::string locateFile(const std::string& filepathSuffix, const std::vector<std::string>& directories)
|
||||
{
|
||||
const int MAX_DEPTH{10};
|
||||
bool found{false};
|
||||
std::string filepath;
|
||||
|
||||
for (auto& dir : directories)
|
||||
{
|
||||
filepath = dir + filepathSuffix;
|
||||
|
||||
for (int i = 0; i < MAX_DEPTH && !found; i++)
|
||||
{
|
||||
std::ifstream checkFile(filepath);
|
||||
found = checkFile.is_open();
|
||||
if (found) break;
|
||||
filepath = "../" + filepath; // Try again in parent dir
|
||||
}
|
||||
|
||||
if (found)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
filepath.clear();
|
||||
}
|
||||
|
||||
if (filepath.empty()) {
|
||||
std::string directoryList = std::accumulate(directories.begin() + 1, directories.end(), directories.front(),
|
||||
[](const std::string& a, const std::string& b) { return a + "\n\t" + b; });
|
||||
throw std::runtime_error("Could not find " + filepathSuffix + " in data directories:\n\t" + directoryList);
|
||||
}
|
||||
return filepath;
|
||||
}
|
||||
|
||||
inline void readPGMFile(const std::string& fileName, uint8_t* buffer, int inH, int inW)
|
||||
{
|
||||
std::ifstream infile(fileName, std::ifstream::binary);
|
||||
assert(infile.is_open() && "Attempting to read from a file that is not open.");
|
||||
std::string magic, h, w, max;
|
||||
infile >> magic >> h >> w >> max;
|
||||
infile.seekg(1, infile.cur);
|
||||
infile.read(reinterpret_cast<char*>(buffer), inH * inW);
|
||||
}
|
||||
|
||||
namespace samples_common
|
||||
{
|
||||
|
||||
inline void* safeCudaMalloc(size_t memSize)
|
||||
{
|
||||
void* deviceMem;
|
||||
CHECK(cudaMalloc(&deviceMem, memSize));
|
||||
if (deviceMem == nullptr)
|
||||
{
|
||||
std::cerr << "Out of memory" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
return deviceMem;
|
||||
}
|
||||
|
||||
inline bool isDebug()
|
||||
{
|
||||
return (std::getenv("TENSORRT_DEBUG") ? true : false);
|
||||
}
|
||||
|
||||
struct InferDeleter
|
||||
{
|
||||
template <typename T>
|
||||
void operator()(T* obj) const
|
||||
{
|
||||
if (obj) {
|
||||
obj->destroy();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
inline std::shared_ptr<T> infer_object(T* obj)
|
||||
{
|
||||
if (!obj) {
|
||||
throw std::runtime_error("Failed to create object");
|
||||
}
|
||||
return std::shared_ptr<T>(obj, InferDeleter());
|
||||
}
|
||||
|
||||
template <class Iter>
|
||||
inline std::vector<size_t> argsort(Iter begin, Iter end, bool reverse = false)
|
||||
{
|
||||
std::vector<size_t> inds(end - begin);
|
||||
std::iota(inds.begin(), inds.end(), 0);
|
||||
if (reverse) {
|
||||
std::sort(inds.begin(), inds.end(), [&begin](size_t i1, size_t i2) {
|
||||
return begin[i2] < begin[i1];
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
std::sort(inds.begin(), inds.end(), [&begin](size_t i1, size_t i2) {
|
||||
return begin[i1] < begin[i2];
|
||||
});
|
||||
}
|
||||
return inds;
|
||||
}
|
||||
|
||||
inline bool readReferenceFile(const std::string& fileName, std::vector<std::string>& refVector)
|
||||
{
|
||||
std::ifstream infile(fileName);
|
||||
if (!infile.is_open()) {
|
||||
cout << "ERROR: readReferenceFile: Attempting to read from a file that is not open." << endl;
|
||||
return false;
|
||||
}
|
||||
std::string line;
|
||||
while (std::getline(infile, line)) {
|
||||
if (line.empty()) continue;
|
||||
refVector.push_back(line);
|
||||
}
|
||||
infile.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename result_vector_t>
|
||||
inline std::vector<std::string> classify(const vector<string>& refVector, const result_vector_t& output, const size_t topK)
|
||||
{
|
||||
auto inds = samples_common::argsort(output.cbegin(), output.cend(), true);
|
||||
std::vector<std::string> result;
|
||||
for (size_t k = 0; k < topK; ++k) {
|
||||
result.push_back(refVector[inds[k]]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
//...LG returns top K indices, not values.
|
||||
template <typename T>
|
||||
inline vector<size_t> topK(const vector<T> inp, const size_t k)
|
||||
{
|
||||
vector<size_t> result;
|
||||
std::vector<size_t> inds = samples_common::argsort(inp.cbegin(), inp.cend(), true);
|
||||
result.assign(inds.begin(), inds.begin()+k);
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool readASCIIFile(const string& fileName, const size_t size, vector<T>& out)
|
||||
{
|
||||
std::ifstream infile(fileName);
|
||||
if (!infile.is_open()) {
|
||||
cout << "ERROR readASCIIFile: Attempting to read from a file that is not open." << endl;
|
||||
return false;
|
||||
}
|
||||
out.clear();
|
||||
out.reserve(size);
|
||||
out.assign(std::istream_iterator<T>(infile), std::istream_iterator<T>());
|
||||
infile.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline bool writeASCIIFile(const string& fileName, const vector<T>& in)
|
||||
{
|
||||
std::ofstream outfile(fileName);
|
||||
if (!outfile.is_open()) {
|
||||
cout << "ERROR: writeASCIIFile: Attempting to write to a file that is not open." << endl;
|
||||
return false;
|
||||
}
|
||||
for (auto fn : in) {
|
||||
outfile << fn << " ";
|
||||
}
|
||||
outfile.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void print_version()
|
||||
{
|
||||
//... This can be only done after statically linking this support into parserONNX.library
|
||||
#if 0
|
||||
std::cout << "Parser built against:" << std::endl;
|
||||
std::cout << " ONNX IR version: " << nvonnxparser::onnx_ir_version_string(onnx::IR_VERSION) << std::endl;
|
||||
#endif
|
||||
std::cout << " TensorRT version: "
|
||||
<< NV_TENSORRT_MAJOR << "."
|
||||
<< NV_TENSORRT_MINOR << "."
|
||||
<< NV_TENSORRT_PATCH << "."
|
||||
<< NV_TENSORRT_BUILD << std::endl;
|
||||
}
|
||||
|
||||
inline string getFileType(const string& filepath)
|
||||
{
|
||||
return filepath.substr(filepath.find_last_of(".") + 1);
|
||||
}
|
||||
|
||||
inline string toLower(const string& inp)
|
||||
{
|
||||
string out = inp;
|
||||
std::transform(out.begin(), out.end(), out.begin(), ::tolower);
|
||||
return out;
|
||||
}
|
||||
|
||||
inline unsigned int getElementSize(nvinfer1::DataType t)
|
||||
{
|
||||
switch (t)
|
||||
{
|
||||
case nvinfer1::DataType::kINT32: return 4;
|
||||
case nvinfer1::DataType::kFLOAT: return 4;
|
||||
case nvinfer1::DataType::kHALF: return 2;
|
||||
case nvinfer1::DataType::kINT8: return 1;
|
||||
}
|
||||
throw std::runtime_error("Invalid DataType.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline int64_t volume(const nvinfer1::Dims& d)
|
||||
{
|
||||
return std::accumulate(d.d, d.d + d.nbDims, 1, std::multiplies<int64_t>());
|
||||
}
|
||||
|
||||
// Struct to maintain command-line arguments.
|
||||
struct Args
|
||||
{
|
||||
bool runInInt8 = false;
|
||||
};
|
||||
|
||||
// Populates the Args struct with the provided command-line parameters.
|
||||
inline void parseArgs(Args& args, int argc, char* argv[])
|
||||
{
|
||||
if (argc >= 1)
|
||||
{
|
||||
for (int i = 1; i < argc; ++i)
|
||||
{
|
||||
if (!strcmp(argv[i], "--int8")) args.runInInt8 = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int C, int H, int W>
|
||||
struct PPM
|
||||
{
|
||||
std::string magic, fileName;
|
||||
int h, w, max;
|
||||
uint8_t buffer[C * H * W];
|
||||
};
|
||||
|
||||
struct BBox
|
||||
{
|
||||
float x1, y1, x2, y2;
|
||||
};
|
||||
|
||||
template <int C, int H, int W>
|
||||
inline void writePPMFileWithBBox(const std::string& filename, PPM<C, H, W>& ppm, const BBox& bbox)
|
||||
{
|
||||
std::ofstream outfile("./" + filename, std::ofstream::binary);
|
||||
assert(!outfile.fail());
|
||||
outfile << "P6" << "\n" << ppm.w << " " << ppm.h << "\n" << ppm.max << "\n";
|
||||
auto round = [](float x) -> int { return int(std::floor(x + 0.5f)); };
|
||||
const int x1 = std::min(std::max(0, round(int(bbox.x1))), W - 1);
|
||||
const int x2 = std::min(std::max(0, round(int(bbox.x2))), W - 1);
|
||||
const int y1 = std::min(std::max(0, round(int(bbox.y1))), H - 1);
|
||||
const int y2 = std::min(std::max(0, round(int(bbox.y2))), H - 1);
|
||||
for (int x = x1; x <= x2; ++x)
|
||||
{
|
||||
// bbox top border
|
||||
ppm.buffer[(y1 * ppm.w + x) * 3] = 255;
|
||||
ppm.buffer[(y1 * ppm.w + x) * 3 + 1] = 0;
|
||||
ppm.buffer[(y1 * ppm.w + x) * 3 + 2] = 0;
|
||||
// bbox bottom border
|
||||
ppm.buffer[(y2 * ppm.w + x) * 3] = 255;
|
||||
ppm.buffer[(y2 * ppm.w + x) * 3 + 1] = 0;
|
||||
ppm.buffer[(y2 * ppm.w + x) * 3 + 2] = 0;
|
||||
}
|
||||
for (int y = y1; y <= y2; ++y)
|
||||
{
|
||||
// bbox left border
|
||||
ppm.buffer[(y * ppm.w + x1) * 3] = 255;
|
||||
ppm.buffer[(y * ppm.w + x1) * 3 + 1] = 0;
|
||||
ppm.buffer[(y * ppm.w + x1) * 3 + 2] = 0;
|
||||
// bbox right border
|
||||
ppm.buffer[(y * ppm.w + x2) * 3] = 255;
|
||||
ppm.buffer[(y * ppm.w + x2) * 3 + 1] = 0;
|
||||
ppm.buffer[(y * ppm.w + x2) * 3 + 2] = 0;
|
||||
}
|
||||
outfile.write(reinterpret_cast<char*>(ppm.buffer), ppm.w * ppm.h * 3);
|
||||
}
|
||||
|
||||
} // namespace samples_common
|
||||
|
||||
#endif // _TRT_COMMON_H_
|
||||
@ -1,27 +0,0 @@
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdio.h>
|
||||
#include "h_sigmoid.cuh"
|
||||
|
||||
|
||||
__global__ void _hSigmoidKer(float const *in, float *out, int size) {
|
||||
int index = threadIdx.x + blockIdx.x * blockDim.x;
|
||||
if (index >= size)
|
||||
return ;
|
||||
|
||||
if (in[index] > 3 )
|
||||
out[index] = 1;
|
||||
else if (in[index] < -3)
|
||||
out[index] = 0;
|
||||
else
|
||||
out[index] = (in[index] + 3)/6;
|
||||
}
|
||||
|
||||
extern "C" void cuh_sigmoid(float const *in, float *out, int size) {
|
||||
int block_size = 256;
|
||||
int grid_size = (size + block_size - 1) / block_size;
|
||||
_hSigmoidKer<<<grid_size, block_size>>>(in, out, size);
|
||||
cudaError_t err = cudaGetLastError();
|
||||
if (err != cudaSuccess) {
|
||||
fprintf(stderr, "Failed to launch _leakyReluKer kernel (error code %s)!\n", cudaGetErrorString(err));
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
#ifndef HSIGMOID_H
|
||||
#define HSIGMOID_H
|
||||
|
||||
extern "C"
|
||||
|
||||
void cuh_sigmoid(float const *in, float *out, int size);
|
||||
|
||||
#endif
|
||||
@ -1,62 +0,0 @@
|
||||
#include "common.h"
|
||||
#include "h_sigmoid.cuh"
|
||||
#include "h_sigmoidplugin.h"
|
||||
|
||||
using namespace nvinfer1;
|
||||
using nvinfer1::HSigmoidPlugin;
|
||||
using nvinfer1::PluginFactory;
|
||||
|
||||
HSigmoidPlugin::HSigmoidPlugin() {
|
||||
}
|
||||
|
||||
HSigmoidPlugin::HSigmoidPlugin(const void* buffer, size_t size) {
|
||||
assert(size == sizeof(input_size_));
|
||||
input_size_ = *reinterpret_cast<const int*>(buffer);
|
||||
}
|
||||
|
||||
int HSigmoidPlugin::getNbOutputs() const {
|
||||
return 1;
|
||||
}
|
||||
|
||||
Dims HSigmoidPlugin::getOutputDimensions(int index, const Dims* inputs, int nbInputDims) {
|
||||
assert(nbInputDims == 1);
|
||||
assert(index == 0);
|
||||
// Output dimensions
|
||||
return DimsCHW(inputs[0].d[0], inputs[0].d[1], inputs[0].d[2]);
|
||||
}
|
||||
|
||||
void HSigmoidPlugin::configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) {
|
||||
input_size_ = inputDims[0].d[0] * inputDims[0].d[1] * inputDims[0].d[2];
|
||||
}
|
||||
|
||||
int HSigmoidPlugin::initialize() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HSigmoidPlugin::terminate() {}
|
||||
|
||||
size_t HSigmoidPlugin::getWorkspaceSize(int maxBatchSize) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int HSigmoidPlugin::enqueue(int batchSize, const void* const* inputs, void** outputs, void* workspace, cudaStream_t stream) {
|
||||
cuh_sigmoid(reinterpret_cast<float const*>(inputs[0]), reinterpret_cast<float*>(outputs[0]), input_size_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
size_t HSigmoidPlugin::getSerializationSize() {
|
||||
return sizeof(input_size_);
|
||||
}
|
||||
|
||||
void HSigmoidPlugin::serialize(void* buffer) {
|
||||
*reinterpret_cast<int*>(buffer) = input_size_;
|
||||
}
|
||||
|
||||
IPlugin* PluginFactory::createPlugin(const char* layerName, const void* serialData, size_t serialLength) {
|
||||
IPlugin *plugin = nullptr;
|
||||
if (strstr(layerName, "h_sigmoid") != NULL) {
|
||||
plugin = new HSigmoidPlugin(serialData, serialLength);
|
||||
}
|
||||
return plugin;
|
||||
}
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
#ifndef HSIGMOID_PLUGIN_H
|
||||
#define HSIGMOID_PLUGIN_H
|
||||
#include <NvInfer.h>
|
||||
|
||||
namespace nvinfer1 {
|
||||
class HSigmoidPlugin : public IPlugin {
|
||||
public:
|
||||
HSigmoidPlugin();
|
||||
HSigmoidPlugin(const void* buffer, size_t size);
|
||||
~HSigmoidPlugin() override = default;
|
||||
int getNbOutputs() const override;
|
||||
Dims getOutputDimensions(int index, const Dims* inputs, int nbInputDims) override;
|
||||
void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) override;
|
||||
int initialize() override;
|
||||
void terminate() override;
|
||||
size_t getWorkspaceSize(int maxBatchSize) const override;
|
||||
int enqueue(
|
||||
int batchSize, const void* const* inputs, void** outputs, void* workspace, cudaStream_t stream) override;
|
||||
size_t getSerializationSize() override;
|
||||
void serialize(void* buffer) override;
|
||||
|
||||
private:
|
||||
int input_size_;
|
||||
};
|
||||
|
||||
class PluginFactory : public IPluginFactory {
|
||||
public:
|
||||
IPlugin* createPlugin(const char* layerName, const void* serialData, size_t serialLength) override;
|
||||
};
|
||||
|
||||
}
|
||||
#endif
|
||||
503
mobilenetv3/logging.h
Normal file
503
mobilenetv3/logging.h
Normal file
@ -0,0 +1,503 @@
|
||||
/*
|
||||
* Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef TENSORRT_LOGGING_H
|
||||
#define TENSORRT_LOGGING_H
|
||||
|
||||
#include "NvInferRuntimeCommon.h"
|
||||
#include <cassert>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <ostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
using Severity = nvinfer1::ILogger::Severity;
|
||||
|
||||
class LogStreamConsumerBuffer : public std::stringbuf
|
||||
{
|
||||
public:
|
||||
LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix, bool shouldLog)
|
||||
: mOutput(stream)
|
||||
, mPrefix(prefix)
|
||||
, mShouldLog(shouldLog)
|
||||
{
|
||||
}
|
||||
|
||||
LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other)
|
||||
: mOutput(other.mOutput)
|
||||
{
|
||||
}
|
||||
|
||||
~LogStreamConsumerBuffer()
|
||||
{
|
||||
// std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence
|
||||
// std::streambuf::pptr() gives a pointer to the current position of the output sequence
|
||||
// if the pointer to the beginning is not equal to the pointer to the current position,
|
||||
// call putOutput() to log the output to the stream
|
||||
if (pbase() != pptr())
|
||||
{
|
||||
putOutput();
|
||||
}
|
||||
}
|
||||
|
||||
// synchronizes the stream buffer and returns 0 on success
|
||||
// synchronizing the stream buffer consists of inserting the buffer contents into the stream,
|
||||
// resetting the buffer and flushing the stream
|
||||
virtual int sync()
|
||||
{
|
||||
putOutput();
|
||||
return 0;
|
||||
}
|
||||
|
||||
void putOutput()
|
||||
{
|
||||
if (mShouldLog)
|
||||
{
|
||||
// prepend timestamp
|
||||
std::time_t timestamp = std::time(nullptr);
|
||||
tm* tm_local = std::localtime(×tamp);
|
||||
std::cout << "[";
|
||||
std::cout << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon << "/";
|
||||
std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/";
|
||||
std::cout << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year << "-";
|
||||
std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":";
|
||||
std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":";
|
||||
std::cout << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] ";
|
||||
// std::stringbuf::str() gets the string contents of the buffer
|
||||
// insert the buffer contents pre-appended by the appropriate prefix into the stream
|
||||
mOutput << mPrefix << str();
|
||||
// set the buffer to empty
|
||||
str("");
|
||||
// flush the stream
|
||||
mOutput.flush();
|
||||
}
|
||||
}
|
||||
|
||||
void setShouldLog(bool shouldLog)
|
||||
{
|
||||
mShouldLog = shouldLog;
|
||||
}
|
||||
|
||||
private:
|
||||
std::ostream& mOutput;
|
||||
std::string mPrefix;
|
||||
bool mShouldLog;
|
||||
};
|
||||
|
||||
//!
|
||||
//! \class LogStreamConsumerBase
|
||||
//! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer
|
||||
//!
|
||||
class LogStreamConsumerBase
|
||||
{
|
||||
public:
|
||||
LogStreamConsumerBase(std::ostream& stream, const std::string& prefix, bool shouldLog)
|
||||
: mBuffer(stream, prefix, shouldLog)
|
||||
{
|
||||
}
|
||||
|
||||
protected:
|
||||
LogStreamConsumerBuffer mBuffer;
|
||||
};
|
||||
|
||||
//!
|
||||
//! \class LogStreamConsumer
|
||||
//! \brief Convenience object used to facilitate use of C++ stream syntax when logging messages.
|
||||
//! Order of base classes is LogStreamConsumerBase and then std::ostream.
|
||||
//! This is because the LogStreamConsumerBase class is used to initialize the LogStreamConsumerBuffer member field
|
||||
//! in LogStreamConsumer and then the address of the buffer is passed to std::ostream.
|
||||
//! This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream.
|
||||
//! Please do not change the order of the parent classes.
|
||||
//!
|
||||
class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream
|
||||
{
|
||||
public:
|
||||
//! \brief Creates a LogStreamConsumer which logs messages with level severity.
|
||||
//! Reportable severity determines if the messages are severe enough to be logged.
|
||||
LogStreamConsumer(Severity reportableSeverity, Severity severity)
|
||||
: LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity)
|
||||
, std::ostream(&mBuffer) // links the stream buffer with the stream
|
||||
, mShouldLog(severity <= reportableSeverity)
|
||||
, mSeverity(severity)
|
||||
{
|
||||
}
|
||||
|
||||
LogStreamConsumer(LogStreamConsumer&& other)
|
||||
: LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog)
|
||||
, std::ostream(&mBuffer) // links the stream buffer with the stream
|
||||
, mShouldLog(other.mShouldLog)
|
||||
, mSeverity(other.mSeverity)
|
||||
{
|
||||
}
|
||||
|
||||
void setReportableSeverity(Severity reportableSeverity)
|
||||
{
|
||||
mShouldLog = mSeverity <= reportableSeverity;
|
||||
mBuffer.setShouldLog(mShouldLog);
|
||||
}
|
||||
|
||||
private:
|
||||
static std::ostream& severityOstream(Severity severity)
|
||||
{
|
||||
return severity >= Severity::kINFO ? std::cout : std::cerr;
|
||||
}
|
||||
|
||||
static std::string severityPrefix(Severity severity)
|
||||
{
|
||||
switch (severity)
|
||||
{
|
||||
case Severity::kINTERNAL_ERROR: return "[F] ";
|
||||
case Severity::kERROR: return "[E] ";
|
||||
case Severity::kWARNING: return "[W] ";
|
||||
case Severity::kINFO: return "[I] ";
|
||||
case Severity::kVERBOSE: return "[V] ";
|
||||
default: assert(0); return "";
|
||||
}
|
||||
}
|
||||
|
||||
bool mShouldLog;
|
||||
Severity mSeverity;
|
||||
};
|
||||
|
||||
//! \class Logger
|
||||
//!
|
||||
//! \brief Class which manages logging of TensorRT tools and samples
|
||||
//!
|
||||
//! \details This class provides a common interface for TensorRT tools and samples to log information to the console,
|
||||
//! and supports logging two types of messages:
|
||||
//!
|
||||
//! - Debugging messages with an associated severity (info, warning, error, or internal error/fatal)
|
||||
//! - Test pass/fail messages
|
||||
//!
|
||||
//! The advantage of having all samples use this class for logging as opposed to emitting directly to stdout/stderr is
|
||||
//! that the logic for controlling the verbosity and formatting of sample output is centralized in one location.
|
||||
//!
|
||||
//! In the future, this class could be extended to support dumping test results to a file in some standard format
|
||||
//! (for example, JUnit XML), and providing additional metadata (e.g. timing the duration of a test run).
|
||||
//!
|
||||
//! TODO: For backwards compatibility with existing samples, this class inherits directly from the nvinfer1::ILogger
|
||||
//! interface, which is problematic since there isn't a clean separation between messages coming from the TensorRT
|
||||
//! library and messages coming from the sample.
|
||||
//!
|
||||
//! In the future (once all samples are updated to use Logger::getTRTLogger() to access the ILogger) we can refactor the
|
||||
//! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger
|
||||
//! object.
|
||||
|
||||
class Logger : public nvinfer1::ILogger
|
||||
{
|
||||
public:
|
||||
Logger(Severity severity = Severity::kWARNING)
|
||||
: mReportableSeverity(severity)
|
||||
{
|
||||
}
|
||||
|
||||
//!
|
||||
//! \enum TestResult
|
||||
//! \brief Represents the state of a given test
|
||||
//!
|
||||
enum class TestResult
|
||||
{
|
||||
kRUNNING, //!< The test is running
|
||||
kPASSED, //!< The test passed
|
||||
kFAILED, //!< The test failed
|
||||
kWAIVED //!< The test was waived
|
||||
};
|
||||
|
||||
//!
|
||||
//! \brief Forward-compatible method for retrieving the nvinfer::ILogger associated with this Logger
|
||||
//! \return The nvinfer1::ILogger associated with this Logger
|
||||
//!
|
||||
//! TODO Once all samples are updated to use this method to register the logger with TensorRT,
|
||||
//! we can eliminate the inheritance of Logger from ILogger
|
||||
//!
|
||||
nvinfer1::ILogger& getTRTLogger()
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Implementation of the nvinfer1::ILogger::log() virtual method
|
||||
//!
|
||||
//! Note samples should not be calling this function directly; it will eventually go away once we eliminate the
|
||||
//! inheritance from nvinfer1::ILogger
|
||||
//!
|
||||
void log(Severity severity, const char* msg) override
|
||||
{
|
||||
LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << std::endl;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Method for controlling the verbosity of logging output
|
||||
//!
|
||||
//! \param severity The logger will only emit messages that have severity of this level or higher.
|
||||
//!
|
||||
void setReportableSeverity(Severity severity)
|
||||
{
|
||||
mReportableSeverity = severity;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Opaque handle that holds logging information for a particular test
|
||||
//!
|
||||
//! This object is an opaque handle to information used by the Logger to print test results.
|
||||
//! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used
|
||||
//! with Logger::reportTest{Start,End}().
|
||||
//!
|
||||
class TestAtom
|
||||
{
|
||||
public:
|
||||
TestAtom(TestAtom&&) = default;
|
||||
|
||||
private:
|
||||
friend class Logger;
|
||||
|
||||
TestAtom(bool started, const std::string& name, const std::string& cmdline)
|
||||
: mStarted(started)
|
||||
, mName(name)
|
||||
, mCmdline(cmdline)
|
||||
{
|
||||
}
|
||||
|
||||
bool mStarted;
|
||||
std::string mName;
|
||||
std::string mCmdline;
|
||||
};
|
||||
|
||||
//!
|
||||
//! \brief Define a test for logging
|
||||
//!
|
||||
//! \param[in] name The name of the test. This should be a string starting with
|
||||
//! "TensorRT" and containing dot-separated strings containing
|
||||
//! the characters [A-Za-z0-9_].
|
||||
//! For example, "TensorRT.sample_googlenet"
|
||||
//! \param[in] cmdline The command line used to reproduce the test
|
||||
//
|
||||
//! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
|
||||
//!
|
||||
static TestAtom defineTest(const std::string& name, const std::string& cmdline)
|
||||
{
|
||||
return TestAtom(false, name, cmdline);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief A convenience overloaded version of defineTest() that accepts an array of command-line arguments
|
||||
//! as input
|
||||
//!
|
||||
//! \param[in] name The name of the test
|
||||
//! \param[in] argc The number of command-line arguments
|
||||
//! \param[in] argv The array of command-line arguments (given as C strings)
|
||||
//!
|
||||
//! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
|
||||
static TestAtom defineTest(const std::string& name, int argc, char const* const* argv)
|
||||
{
|
||||
auto cmdline = genCmdlineString(argc, argv);
|
||||
return defineTest(name, cmdline);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Report that a test has started.
|
||||
//!
|
||||
//! \pre reportTestStart() has not been called yet for the given testAtom
|
||||
//!
|
||||
//! \param[in] testAtom The handle to the test that has started
|
||||
//!
|
||||
static void reportTestStart(TestAtom& testAtom)
|
||||
{
|
||||
reportTestResult(testAtom, TestResult::kRUNNING);
|
||||
assert(!testAtom.mStarted);
|
||||
testAtom.mStarted = true;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief Report that a test has ended.
|
||||
//!
|
||||
//! \pre reportTestStart() has been called for the given testAtom
|
||||
//!
|
||||
//! \param[in] testAtom The handle to the test that has ended
|
||||
//! \param[in] result The result of the test. Should be one of TestResult::kPASSED,
|
||||
//! TestResult::kFAILED, TestResult::kWAIVED
|
||||
//!
|
||||
static void reportTestEnd(const TestAtom& testAtom, TestResult result)
|
||||
{
|
||||
assert(result != TestResult::kRUNNING);
|
||||
assert(testAtom.mStarted);
|
||||
reportTestResult(testAtom, result);
|
||||
}
|
||||
|
||||
static int reportPass(const TestAtom& testAtom)
|
||||
{
|
||||
reportTestEnd(testAtom, TestResult::kPASSED);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
static int reportFail(const TestAtom& testAtom)
|
||||
{
|
||||
reportTestEnd(testAtom, TestResult::kFAILED);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
static int reportWaive(const TestAtom& testAtom)
|
||||
{
|
||||
reportTestEnd(testAtom, TestResult::kWAIVED);
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
|
||||
static int reportTest(const TestAtom& testAtom, bool pass)
|
||||
{
|
||||
return pass ? reportPass(testAtom) : reportFail(testAtom);
|
||||
}
|
||||
|
||||
Severity getReportableSeverity() const
|
||||
{
|
||||
return mReportableSeverity;
|
||||
}
|
||||
|
||||
private:
|
||||
//!
|
||||
//! \brief returns an appropriate string for prefixing a log message with the given severity
|
||||
//!
|
||||
static const char* severityPrefix(Severity severity)
|
||||
{
|
||||
switch (severity)
|
||||
{
|
||||
case Severity::kINTERNAL_ERROR: return "[F] ";
|
||||
case Severity::kERROR: return "[E] ";
|
||||
case Severity::kWARNING: return "[W] ";
|
||||
case Severity::kINFO: return "[I] ";
|
||||
case Severity::kVERBOSE: return "[V] ";
|
||||
default: assert(0); return "";
|
||||
}
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief returns an appropriate string for prefixing a test result message with the given result
|
||||
//!
|
||||
static const char* testResultString(TestResult result)
|
||||
{
|
||||
switch (result)
|
||||
{
|
||||
case TestResult::kRUNNING: return "RUNNING";
|
||||
case TestResult::kPASSED: return "PASSED";
|
||||
case TestResult::kFAILED: return "FAILED";
|
||||
case TestResult::kWAIVED: return "WAIVED";
|
||||
default: assert(0); return "";
|
||||
}
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief returns an appropriate output stream (cout or cerr) to use with the given severity
|
||||
//!
|
||||
static std::ostream& severityOstream(Severity severity)
|
||||
{
|
||||
return severity >= Severity::kINFO ? std::cout : std::cerr;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief method that implements logging test results
|
||||
//!
|
||||
static void reportTestResult(const TestAtom& testAtom, TestResult result)
|
||||
{
|
||||
severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # "
|
||||
<< testAtom.mCmdline << std::endl;
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief generate a command line string from the given (argc, argv) values
|
||||
//!
|
||||
static std::string genCmdlineString(int argc, char const* const* argv)
|
||||
{
|
||||
std::stringstream ss;
|
||||
for (int i = 0; i < argc; i++)
|
||||
{
|
||||
if (i > 0)
|
||||
ss << " ";
|
||||
ss << argv[i];
|
||||
}
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
Severity mReportableSeverity;
|
||||
};
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_VERBOSE(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_VERBOSE(const Logger& logger)
|
||||
{
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINFO
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_INFO(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_INFO(const Logger& logger)
|
||||
{
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kWARNING
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_WARN(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_WARN(const Logger& logger)
|
||||
{
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kERROR
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_ERROR(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_ERROR(const Logger& logger)
|
||||
{
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR);
|
||||
}
|
||||
|
||||
//!
|
||||
//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINTERNAL_ERROR
|
||||
// ("fatal" severity)
|
||||
//!
|
||||
//! Example usage:
|
||||
//!
|
||||
//! LOG_FATAL(logger) << "hello world" << std::endl;
|
||||
//!
|
||||
inline LogStreamConsumer LOG_FATAL(const Logger& logger)
|
||||
{
|
||||
return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINTERNAL_ERROR);
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
#endif // TENSORRT_LOGGING_H
|
||||
@ -1,15 +1,24 @@
|
||||
#include "NvInfer.h"
|
||||
#include "cuda_runtime_api.h"
|
||||
#include "common.h"
|
||||
#include "logging.h"
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
#include <chrono>
|
||||
//#include "plugin_factory.h"
|
||||
#include "h_sigmoidplugin.h"
|
||||
//#include "leakyplugin.h"
|
||||
#include <cmath>
|
||||
|
||||
#define CHECK(status) \
|
||||
do\
|
||||
{\
|
||||
auto ret = (status);\
|
||||
if (ret != 0)\
|
||||
{\
|
||||
std::cerr << "Cuda failure: " << ret << std::endl;\
|
||||
abort();\
|
||||
}\
|
||||
} while (0)
|
||||
|
||||
// stuff we know about the network and the input/output blobs
|
||||
static const int INPUT_H = 224;
|
||||
@ -101,25 +110,22 @@ IScaleLayer* addBatchNorm(INetworkDefinition *network, std::map<std::string, Wei
|
||||
}
|
||||
|
||||
ILayer* hSwish(INetworkDefinition *network, ITensor& input, std::string name) {
|
||||
//auto hsg = new LeakyPlugin();
|
||||
auto hsg = new HSigmoidPlugin();
|
||||
ITensor* inputTensors[] = {&input};
|
||||
auto hs1 = network->addPlugin(inputTensors,1,*hsg);
|
||||
assert(hs1);
|
||||
hs1->setName(("h_sigmoid"+name).c_str());
|
||||
ILayer* hsw = network->addElementWise(input, *hs1->getOutput(0),ElementWiseOperation::kPROD);
|
||||
auto hsig = network->addActivation(input, ActivationType::kHARD_SIGMOID);
|
||||
assert(hsig);
|
||||
hsig->setAlpha(1.0 / 6.0);
|
||||
hsig->setBeta(0.5);
|
||||
ILayer* hsw = network->addElementWise(input, *hsig->getOutput(0),ElementWiseOperation::kPROD);
|
||||
assert(hsw);
|
||||
return hsw;
|
||||
}
|
||||
|
||||
|
||||
ILayer* convBnHswish(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, int outch, int ksize, int s, int g, std::string lname) {
|
||||
Weights emptywts{DataType::kFLOAT, nullptr, 0};
|
||||
int p = (ksize - 1) / 2;
|
||||
IConvolutionLayer* conv1 = network->addConvolution(input, outch, DimsHW{ksize, ksize}, weightMap[lname + "0.weight"], emptywts);
|
||||
IConvolutionLayer* conv1 = network->addConvolutionNd(input, outch, DimsHW{ksize, ksize}, weightMap[lname + "0.weight"], emptywts);
|
||||
assert(conv1);
|
||||
conv1->setStride(DimsHW{s, s});
|
||||
conv1->setPadding(DimsHW{p, p});
|
||||
conv1->setStrideNd(DimsHW{s, s});
|
||||
conv1->setPaddingNd(DimsHW{p, p});
|
||||
conv1->setNbGroups(g);
|
||||
|
||||
IScaleLayer* bn1 = addBatchNorm(network, weightMap, *conv1->getOutput(0), lname + "1", 1e-5);
|
||||
@ -130,18 +136,19 @@ ILayer* convBnHswish(INetworkDefinition *network, std::map<std::string, Weights>
|
||||
|
||||
ILayer* seLayer(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, int c, int w, std::string lname) {
|
||||
int h = w;
|
||||
IPoolingLayer* l1 = network->addPooling(input,PoolingType::kAVERAGE,DimsHW(w, h));
|
||||
IPoolingLayer* l1 = network->addPoolingNd(input, PoolingType::kAVERAGE, DimsHW(w, h));
|
||||
assert(l1);
|
||||
l1->setStride(DimsHW{w, h});
|
||||
IFullyConnectedLayer* l2 = network->addFullyConnected(*l1->getOutput(0), BS*c/4,weightMap[lname+"fc.0.weight"],weightMap[lname+"fc.0.bias"]);
|
||||
IActivationLayer* relu1 = network->addActivation(*l2->getOutput(0),ActivationType::kRELU);
|
||||
IFullyConnectedLayer* l4 = network->addFullyConnected(*relu1->getOutput(0), BS*c,weightMap[lname+"fc.2.weight"],weightMap[lname+"fc.2.bias"]);
|
||||
auto hsg = new HSigmoidPlugin();
|
||||
ITensor* inputTensors[] = {l4->getOutput(0)};
|
||||
auto hs1 = network->addPlugin(inputTensors,1,*hsg);
|
||||
assert(hs1);
|
||||
hs1->setName(("h_sigmoid"+lname + "seLayer").c_str());
|
||||
ILayer* se = network->addElementWise(input, *hs1->getOutput(0), ElementWiseOperation::kPROD);
|
||||
l1->setStrideNd(DimsHW{w, h});
|
||||
IFullyConnectedLayer* l2 = network->addFullyConnected(*l1->getOutput(0), BS*c/4, weightMap[lname+"fc.0.weight"], weightMap[lname+"fc.0.bias"]);
|
||||
IActivationLayer* relu1 = network->addActivation(*l2->getOutput(0), ActivationType::kRELU);
|
||||
IFullyConnectedLayer* l4 = network->addFullyConnected(*relu1->getOutput(0), BS*c, weightMap[lname+"fc.2.weight"], weightMap[lname+"fc.2.bias"]);
|
||||
|
||||
auto hsig = network->addActivation(*l4->getOutput(0), ActivationType::kHARD_SIGMOID);
|
||||
assert(hsig);
|
||||
hsig->setAlpha(1.0 / 6.0);
|
||||
hsig->setBeta(0.5);
|
||||
|
||||
ILayer* se = network->addElementWise(input, *hsig->getOutput(0), ElementWiseOperation::kPROD);
|
||||
assert(se);
|
||||
return se;
|
||||
}
|
||||
@ -149,9 +156,9 @@ ILayer* seLayer(INetworkDefinition *network, std::map<std::string, Weights>& wei
|
||||
ILayer* convSeq1(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, int output, int hdim, int k, int s, bool use_se, bool use_hs, int w, std::string lname) {
|
||||
Weights emptywts{DataType::kFLOAT, nullptr, 0};
|
||||
int p = (k - 1) / 2;
|
||||
IConvolutionLayer* conv1 = network->addConvolution(input, hdim, DimsHW{k, k}, weightMap[lname + "0.weight"], emptywts);
|
||||
conv1->setStride(DimsHW{s, s});
|
||||
conv1->setPadding(DimsHW{p, p});
|
||||
IConvolutionLayer* conv1 = network->addConvolutionNd(input, hdim, DimsHW{k, k}, weightMap[lname + "0.weight"], emptywts);
|
||||
conv1->setStrideNd(DimsHW{s, s});
|
||||
conv1->setPaddingNd(DimsHW{p, p});
|
||||
conv1->setNbGroups(hdim);
|
||||
|
||||
IScaleLayer* bn1 = addBatchNorm(network, weightMap, *conv1->getOutput(0), lname + "1", 1e-5);
|
||||
@ -161,27 +168,26 @@ ILayer* convSeq1(INetworkDefinition *network, std::map<std::string, Weights>& we
|
||||
if (use_hs) {
|
||||
ILayer* hsw = hSwish(network, *bn1->getOutput(0), lname+"2");
|
||||
tensor3 = hsw->getOutput(0);
|
||||
}
|
||||
else {
|
||||
IActivationLayer* relu1 = network->addActivation(*bn1->getOutput(0),ActivationType::kRELU);
|
||||
} else {
|
||||
IActivationLayer* relu1 = network->addActivation(*bn1->getOutput(0), ActivationType::kRELU);
|
||||
tensor3 = relu1->getOutput(0);
|
||||
}
|
||||
if (use_se) {
|
||||
ILayer* se1 = seLayer(network, weightMap, *tensor3, hdim, w, lname + "3.");
|
||||
tensor4 = se1->getOutput(0);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
tensor4 = tensor3;
|
||||
}
|
||||
IConvolutionLayer* conv2 = network->addConvolution(*tensor4, output, DimsHW{1, 1}, weightMap[lname + "4.weight"], emptywts);
|
||||
IConvolutionLayer* conv2 = network->addConvolutionNd(*tensor4, output, DimsHW{1, 1}, weightMap[lname + "4.weight"], emptywts);
|
||||
IScaleLayer* bn2 = addBatchNorm(network, weightMap, *conv2->getOutput(0), lname + "5", 1e-5);
|
||||
assert(bn2);
|
||||
return bn2;
|
||||
}
|
||||
|
||||
ILayer* convSeq2(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, int output, int hdim, int k, int s, bool use_se, bool use_hs, int w, std::string lname) {
|
||||
Weights emptywts{DataType::kFLOAT, nullptr, 0};
|
||||
int p = (k - 1) / 2;
|
||||
IConvolutionLayer* conv1 = network->addConvolution(input, hdim, DimsHW{1, 1}, weightMap[lname + "0.weight"], emptywts);
|
||||
IConvolutionLayer* conv1 = network->addConvolutionNd(input, hdim, DimsHW{1, 1}, weightMap[lname + "0.weight"], emptywts);
|
||||
IScaleLayer* bn1 = addBatchNorm(network, weightMap, *conv1->getOutput(0), lname + "1", 1e-5);
|
||||
ITensor *tensor3, *tensor6, *tensor7;
|
||||
tensor3 = nullptr;
|
||||
@ -190,44 +196,40 @@ ILayer* convSeq2(INetworkDefinition *network, std::map<std::string, Weights>& we
|
||||
if (use_hs) {
|
||||
ILayer* hsw1 = hSwish(network, *bn1->getOutput(0), lname + "2");
|
||||
tensor3 = hsw1->getOutput(0);
|
||||
}
|
||||
else {
|
||||
IActivationLayer* relu1 = network->addActivation(*bn1->getOutput(0),ActivationType::kRELU);
|
||||
} else {
|
||||
IActivationLayer* relu1 = network->addActivation(*bn1->getOutput(0), ActivationType::kRELU);
|
||||
tensor3 = relu1->getOutput(0);
|
||||
}
|
||||
IConvolutionLayer* conv2 = network->addConvolution(*tensor3, hdim, DimsHW{k, k}, weightMap[lname + "3.weight"], emptywts);
|
||||
conv2->setStride(DimsHW{s, s});
|
||||
conv2->setPadding(DimsHW{p, p});
|
||||
IConvolutionLayer* conv2 = network->addConvolutionNd(*tensor3, hdim, DimsHW{k, k}, weightMap[lname + "3.weight"], emptywts);
|
||||
conv2->setStrideNd(DimsHW{s, s});
|
||||
conv2->setPaddingNd(DimsHW{p, p});
|
||||
conv2->setNbGroups(hdim);
|
||||
IScaleLayer* bn2 = addBatchNorm(network, weightMap, *conv2->getOutput(0), lname + "4", 1e-5);
|
||||
if (use_se) {
|
||||
ILayer* se1 = seLayer(network, weightMap, *bn2->getOutput(0), hdim, w, lname + "5.");
|
||||
tensor6 = se1->getOutput(0);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
tensor6 = bn2->getOutput(0);
|
||||
}
|
||||
if (use_hs) {
|
||||
ILayer* hsw2 = hSwish(network, *tensor6, lname + "6");
|
||||
tensor7 = hsw2->getOutput(0);
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
IActivationLayer* relu2 = network->addActivation(*tensor6, ActivationType::kRELU);
|
||||
tensor7 = relu2->getOutput(0);
|
||||
}
|
||||
IConvolutionLayer* conv3 = network->addConvolution(*tensor7, output, DimsHW{1, 1}, weightMap[lname + "7.weight"], emptywts);
|
||||
IConvolutionLayer* conv3 = network->addConvolutionNd(*tensor7, output, DimsHW{1, 1}, weightMap[lname + "7.weight"], emptywts);
|
||||
IScaleLayer* bn3 = addBatchNorm(network, weightMap, *conv3->getOutput(0), lname + "8", 1e-5);
|
||||
assert(bn3);
|
||||
return bn3;
|
||||
}
|
||||
ILayer* invertedRes(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, std::string lname,
|
||||
int inch, int outch, int s, int hidden, int k, bool use_se, bool use_hs, int w) {
|
||||
|
||||
ILayer* invertedRes(INetworkDefinition *network, std::map<std::string, Weights>& weightMap, ITensor& input, std::string lname, int inch, int outch, int s, int hidden, int k, bool use_se, bool use_hs, int w) {
|
||||
bool use_res_connect = (s == 1 && inch == outch);
|
||||
ILayer *conv = nullptr;
|
||||
if (inch == hidden) {
|
||||
conv = convSeq1(network, weightMap, input, outch, hidden, k, s, use_se, use_hs, w, lname + "conv.");
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
conv = convSeq2(network, weightMap, input, outch, hidden, k, s, use_se, use_hs, w, lname + "conv.");
|
||||
}
|
||||
|
||||
@ -238,11 +240,10 @@ ILayer* invertedRes(INetworkDefinition *network, std::map<std::string, Weights>&
|
||||
}
|
||||
|
||||
// Creat the engine using only the API and not any parser.
|
||||
ICudaEngine* createEngineSmall(unsigned int maxBatchSize, IBuilder* builder, DataType dt)
|
||||
ICudaEngine* createEngineSmall(unsigned int maxBatchSize, IBuilder* builder, IBuilderConfig* config, DataType dt)
|
||||
{
|
||||
INetworkDefinition* network = builder->createNetwork();
|
||||
INetworkDefinition* network = builder->createNetworkV2(0U);
|
||||
|
||||
// Create input tensor of shape { 1, 1, 32, 32 } with name INPUT_BLOB_NAME
|
||||
ITensor* data = network->addInput(INPUT_BLOB_NAME, dt, Dims3{3, INPUT_H, INPUT_W});
|
||||
assert(data);
|
||||
|
||||
@ -265,9 +266,9 @@ ICudaEngine* createEngineSmall(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
ILayer* ew2 = convBnHswish(network, weightMap, *ir11->getOutput(0), 576, 1, 1, 1, "conv.0.");
|
||||
ILayer* se1 = seLayer(network, weightMap, *ew2->getOutput(0), 576, 7, "conv.1.");
|
||||
|
||||
IPoolingLayer* pool1 = network->addPooling(*se1->getOutput(0), PoolingType::kAVERAGE, DimsHW{7, 7});
|
||||
IPoolingLayer* pool1 = network->addPoolingNd(*se1->getOutput(0), PoolingType::kAVERAGE, DimsHW{7, 7});
|
||||
assert(pool1);
|
||||
pool1->setStride(DimsHW{7, 7});
|
||||
pool1->setStrideNd(DimsHW{7, 7});
|
||||
ILayer* sw1 = hSwish(network, *pool1->getOutput(0), "hSwish.0");
|
||||
|
||||
IFullyConnectedLayer* fc1 = network->addFullyConnected(*sw1->getOutput(0), 1280, weightMap["classifier.0.weight"], weightMap["classifier.0.bias"]);
|
||||
@ -284,8 +285,8 @@ ICudaEngine* createEngineSmall(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
|
||||
// Build engine
|
||||
builder->setMaxBatchSize(maxBatchSize);
|
||||
builder->setMaxWorkspaceSize(1 << 20);
|
||||
ICudaEngine* engine = builder->buildCudaEngine(*network);
|
||||
config->setMaxWorkspaceSize(1 << 20);
|
||||
ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config);
|
||||
std::cout << "build out" << std::endl;
|
||||
|
||||
// Don't need the network any more
|
||||
@ -300,11 +301,10 @@ ICudaEngine* createEngineSmall(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
return engine;
|
||||
}
|
||||
|
||||
ICudaEngine* createEngineLarge(unsigned int maxBatchSize, IBuilder* builder, DataType dt)
|
||||
ICudaEngine* createEngineLarge(unsigned int maxBatchSize, IBuilder* builder, IBuilderConfig* config, DataType dt)
|
||||
{
|
||||
INetworkDefinition* network = builder->createNetwork();
|
||||
INetworkDefinition* network = builder->createNetworkV2(0U);
|
||||
|
||||
// Create input tensor of shape { 1, 1, 32, 32 } with name INPUT_BLOB_NAME
|
||||
ITensor* data = network->addInput(INPUT_BLOB_NAME, dt, Dims3{3, INPUT_H, INPUT_W});
|
||||
assert(data);
|
||||
|
||||
@ -330,9 +330,9 @@ ICudaEngine* createEngineLarge(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
auto ir15 = invertedRes(network, weightMap, *ir14->getOutput(0), "features.15.", 160, 160, 1, 960, 5, 1, 1, 7);
|
||||
ILayer* ew2 = convBnHswish(network, weightMap, *ir15->getOutput(0), 960, 1, 1, 1, "conv.0.");
|
||||
|
||||
IPoolingLayer* pool1 = network->addPooling(*ew2->getOutput(0), PoolingType::kAVERAGE, DimsHW{7, 7});
|
||||
IPoolingLayer* pool1 = network->addPoolingNd(*ew2->getOutput(0), PoolingType::kAVERAGE, DimsHW{7, 7});
|
||||
assert(pool1);
|
||||
pool1->setStride(DimsHW{7, 7});
|
||||
pool1->setStrideNd(DimsHW{7, 7});
|
||||
ILayer* sw1 = hSwish(network, *pool1->getOutput(0), "hSwish.0");
|
||||
|
||||
IFullyConnectedLayer* fc1 = network->addFullyConnected(*sw1->getOutput(0), 1280, weightMap["classifier.0.weight"], weightMap["classifier.0.bias"]);
|
||||
@ -346,8 +346,8 @@ ICudaEngine* createEngineLarge(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
|
||||
// Build engine
|
||||
builder->setMaxBatchSize(maxBatchSize);
|
||||
builder->setMaxWorkspaceSize(1 << 20);
|
||||
ICudaEngine* engine = builder->buildCudaEngine(*network);
|
||||
config->setMaxWorkspaceSize(1 << 20);
|
||||
ICudaEngine* engine = builder->buildEngineWithConfig(*network, *config);
|
||||
std::cout << "build out" << std::endl;
|
||||
|
||||
// Don't need the network any more
|
||||
@ -361,20 +361,21 @@ ICudaEngine* createEngineLarge(unsigned int maxBatchSize, IBuilder* builder, Dat
|
||||
|
||||
return engine;
|
||||
}
|
||||
|
||||
void APIToModel(unsigned int maxBatchSize, IHostMemory** modelStream, std::string mode)
|
||||
{
|
||||
// Create builder
|
||||
IBuilder* builder = createInferBuilder(gLogger);
|
||||
IBuilderConfig* config = builder->createBuilderConfig();
|
||||
|
||||
// Create model to populate the network, then set the outputs and create an engine
|
||||
ICudaEngine* engine;
|
||||
|
||||
if (mode == "small") {
|
||||
std::cout << "create engine small" << std::endl;
|
||||
engine = createEngineSmall(maxBatchSize, builder, DataType::kFLOAT);
|
||||
}
|
||||
else if (mode == "large") {
|
||||
engine = createEngineLarge(maxBatchSize, builder, DataType::kFLOAT);
|
||||
engine = createEngineSmall(maxBatchSize, builder, config, DataType::kFLOAT);
|
||||
} else if (mode == "large") {
|
||||
engine = createEngineLarge(maxBatchSize, builder, config, DataType::kFLOAT);
|
||||
}
|
||||
assert(engine != nullptr);
|
||||
|
||||
@ -384,6 +385,7 @@ void APIToModel(unsigned int maxBatchSize, IHostMemory** modelStream, std::strin
|
||||
// Close everything down
|
||||
engine->destroy();
|
||||
builder->destroy();
|
||||
config->destroy();
|
||||
}
|
||||
|
||||
void doInference(IExecutionContext& context, float* input, float* output, int batchSize)
|
||||
@ -466,22 +468,21 @@ int main(int argc, char** argv)
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
// Subtract mean from image
|
||||
float data[3 * INPUT_H * INPUT_W];
|
||||
static float data[3 * INPUT_H * INPUT_W];
|
||||
for (int i = 0; i < 3 * INPUT_H * INPUT_W; i++)
|
||||
data[i] = 1.0;
|
||||
PluginFactory pf;
|
||||
IRuntime* runtime = createInferRuntime(gLogger);
|
||||
assert(runtime != nullptr);
|
||||
ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size, &pf);
|
||||
ICudaEngine* engine = runtime->deserializeCudaEngine(trtModelStream, size);
|
||||
assert(engine != nullptr);
|
||||
IExecutionContext* context = engine->createExecutionContext();
|
||||
assert(context != nullptr);
|
||||
delete[] trtModelStream;
|
||||
|
||||
// Run inference
|
||||
float prob[OUTPUT_SIZE];
|
||||
for (int i = 0; i < 100; i++) {
|
||||
static float prob[OUTPUT_SIZE];
|
||||
for (int i = 0; i < 10; i++) {
|
||||
auto start = std::chrono::system_clock::now();
|
||||
doInference(*context, data, prob, 1);
|
||||
auto end = std::chrono::system_clock::now();
|
||||
@ -495,7 +496,7 @@ int main(int argc, char** argv)
|
||||
|
||||
// Print histogram of the output distribution
|
||||
std::cout << "\nOutput:\n\n";
|
||||
for (unsigned int i = 0; i < 20; i++)
|
||||
for (unsigned int i = 0; i < OUTPUT_SIZE; i++)
|
||||
{
|
||||
std::cout << prob[i] << ", ";
|
||||
//if (i % 10 == 0) std::cout << i / 10 << std::endl;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user