Abstract class for loading a tensor network with TensorRT. More...

#include <tensorNet.h>

Inheritance diagram for tensorNet:

Classes
class	Logger
	Logger class for GIE info/warning/errors. More...

struct	outputLayer

class	Profiler
	Profiler interface for measuring layer timings. More...

Public Member Functions
virtual	~tensorNet ()
	Destory. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean=NULL, const char input_blob="data", const char output_blob="prob", uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean, const char input_blob, const std::vector< std::string > &output_blobs, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator *calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance with multiple output layers. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean, const char input_blob, const Dims3 &input_dims, const std::vector< std::string > &output_blobs, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator *calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance (this variant is used for UFF models) More...

void	EnableLayerProfiler ()
	Manually enable layer profiling times. More...

void	EnableDebug ()
	Manually enable debug messages and synchronization. More...

bool	AllowGPUFallback () const
	Return true if GPU fallback is enabled. More...

deviceType	GetDevice () const
	Retrieve the device being used for execution. More...

precisionType	GetPrecision () const
	Retrieve the type of precision being used. More...

bool	IsPrecision (precisionType type) const
	Check if a particular precision is being used. More...

cudaStream_t	GetStream () const
	Retrieve the stream that the device is operating on. More...

cudaStream_t	CreateStream (bool nonBlocking=true)
	Create and use a new stream for execution. More...

void	SetStream (cudaStream_t stream)
	Set the stream that the device is operating on. More...

const char *	GetPrototxtPath () const
	Retrieve the path to the network prototxt file. More...

const char *	GetModelPath () const
	Retrieve the path to the network model file. More...

modelType	GetModelType () const
	Retrieve the format of the network model. More...

bool	IsModelType (modelType type) const
	Return true if the model is of the specified format. More...

float	GetNetworkTime ()
	Retrieve the network runtime (in milliseconds). More...

float2	GetProfilerTime (profilerQuery query)
	Retrieve the profiler runtime (in milliseconds). More...

float	GetProfilerTime (profilerQuery query, profilerDevice device)
	Retrieve the profiler runtime (in milliseconds). More...

void	PrintProfilerTimes ()
	Print the profiler times (in millseconds). More...

Static Public Member Functions
static precisionType	FindFastestPrecision (deviceType device=DEVICE_GPU, bool allowInt8=true)
	Determine the fastest native precision on a device. More...

static std::vector< precisionType >	DetectNativePrecisions (deviceType device=DEVICE_GPU)
	Detect the precisions supported natively on a device. More...

static bool	DetectNativePrecision (const std::vector< precisionType > &nativeTypes, precisionType type)
	Detect if a particular precision is supported natively. More...

static bool	DetectNativePrecision (precisionType precision, deviceType device=DEVICE_GPU)
	Detect if a particular precision is supported natively. More...

Protected Member Functions
	tensorNet ()
	Constructor. More...

bool	ProfileModel (const std::string &deployFile, const std::string &modelFile, const char input, const Dims3 &inputDims, const std::vector< std::string > &outputs, uint32_t maxBatchSize, precisionType precision, deviceType device, bool allowGPUFallback, nvinfer1::IInt8Calibrator calibrator, std::ostream &modelStream)
	Create and output an optimized network model. More...

void	PROFILER_BEGIN (profilerQuery query)
	Begin a profiling query, before network is run. More...

void	PROFILER_END (profilerQuery query)
	End a profiling query, after the network is run. More...

bool	PROFILER_QUERY (profilerQuery query)
	Query the CUDA part of a profiler query. More...

Protected Attributes
tensorNet::Logger	gLogger

tensorNet::Profiler	gProfiler

std::string	mPrototxtPath

std::string	mModelPath

std::string	mMeanPath

std::string	mInputBlobName

std::string	mCacheEnginePath

std::string	mCacheCalibrationPath

deviceType	mDevice

precisionType	mPrecision

modelType	mModelType

cudaStream_t	mStream

cudaEvent_t	mEventsGPU [PROFILER_TOTAL *2]

timespec	mEventsCPU [PROFILER_TOTAL *2]

nvinfer1::IRuntime *	mInfer

nvinfer1::ICudaEngine *	mEngine

nvinfer1::IExecutionContext *	mContext

uint32_t	mWidth

uint32_t	mHeight

uint32_t	mInputSize

float *	mInputCPU

float *	mInputCUDA

float2	mProfilerTimes [PROFILER_TOTAL+1]

uint32_t	mProfilerQueriesUsed

uint32_t	mProfilerQueriesDone

uint32_t	mMaxBatchSize

bool	mEnableProfiler

bool	mEnableDebug

bool	mAllowGPUFallback

Dims3	mInputDims

std::vector< outputLayer >	mOutputs

Detailed Description

Abstract class for loading a tensor network with TensorRT.

For example implementations,

See also: imageNet and; detectNet

Constructor & Destructor Documentation

◆ ~tensorNet()

virtual tensorNet::~tensorNet ( )

virtual

Destory.

◆ tensorNet()

tensorNet::tensorNet ( )

protected

Constructor.

Member Function Documentation

◆ AllowGPUFallback()

bool tensorNet::AllowGPUFallback ( ) const

inline

Return true if GPU fallback is enabled.

◆ CreateStream()

cudaStream_t tensorNet::CreateStream ( bool nonBlocking = true )

Create and use a new stream for execution.

◆ DetectNativePrecision() [1/2]

static bool tensorNet::DetectNativePrecision	(	const std::vector< precisionType > &	nativeTypes,
		precisionType	type
	)

static

Detect if a particular precision is supported natively.

◆ DetectNativePrecision() [2/2]

static bool tensorNet::DetectNativePrecision	(	precisionType	precision,
		deviceType	device = `DEVICE_GPU`
	)

static

Detect if a particular precision is supported natively.

◆ DetectNativePrecisions()

static std::vector<precisionType> tensorNet::DetectNativePrecisions ( deviceType device = DEVICE_GPU )

static

Detect the precisions supported natively on a device.

◆ EnableDebug()

void tensorNet::EnableDebug ( )

Manually enable debug messages and synchronization.

◆ EnableLayerProfiler()

void tensorNet::EnableLayerProfiler ( )

Manually enable layer profiling times.

◆ FindFastestPrecision()

static precisionType tensorNet::FindFastestPrecision	(	deviceType	device = `DEVICE_GPU`,
		bool	allowInt8 = `true`
	)

static

Determine the fastest native precision on a device.

◆ GetDevice()

deviceType tensorNet::GetDevice ( ) const

inline

Retrieve the device being used for execution.

◆ GetModelPath()

const char* tensorNet::GetModelPath ( ) const

inline

Retrieve the path to the network model file.

◆ GetModelType()

modelType tensorNet::GetModelType ( ) const

inline

Retrieve the format of the network model.

◆ GetNetworkTime()

float tensorNet::GetNetworkTime ( )

inline

Retrieve the network runtime (in milliseconds).

◆ GetPrecision()

precisionType tensorNet::GetPrecision ( ) const

inline

Retrieve the type of precision being used.

◆ GetProfilerTime() [1/2]

float2 tensorNet::GetProfilerTime ( profilerQuery query )

inline

Retrieve the profiler runtime (in milliseconds).

◆ GetProfilerTime() [2/2]

float tensorNet::GetProfilerTime	(	profilerQuery	query,
		profilerDevice	device
	)

inline

Retrieve the profiler runtime (in milliseconds).

◆ GetPrototxtPath()

const char* tensorNet::GetPrototxtPath ( ) const

inline

Retrieve the path to the network prototxt file.

◆ GetStream()

cudaStream_t tensorNet::GetStream ( ) const

inline

Retrieve the stream that the device is operating on.

◆ IsModelType()

bool tensorNet::IsModelType ( modelType type ) const

inline

Return true if the model is of the specified format.

◆ IsPrecision()

bool tensorNet::IsPrecision ( precisionType type ) const

inline

Check if a particular precision is being used.

◆ LoadNetwork() [1/3]

bool tensorNet::LoadNetwork	(	const char *	prototxt,
		const char *	model,
		const char *	mean = `NULL`,
		const char *	input_blob = `"data"`,
		const char *	output_blob = `"prob"`,
		uint32_t	maxBatchSize = `DEFAULT_MAX_BATCH_SIZE`,
		precisionType	precision = `TYPE_FASTEST`,
		deviceType	device = `DEVICE_GPU`,
		bool	allowGPUFallback = `true`,
		nvinfer1::IInt8Calibrator *	calibrator = `NULL`,
		cudaStream_t	stream = `NULL`
	)

Load a new network instance.

Parameters

prototxt	File path to the deployable network prototxt
model	File path to the caffemodel
mean	File path to the mean value binary proto (NULL if none)
input_blob	The name of the input blob data to the network.
output_blob	The name of the output blob data from the network.
maxBatchSize	The maximum batch size that the network will be optimized for.

◆ LoadNetwork() [2/3]

bool tensorNet::LoadNetwork	(	const char *	prototxt,
		const char *	model,
		const char *	mean,
		const char *	input_blob,
		const std::vector< std::string > &	output_blobs,
		uint32_t	maxBatchSize = `DEFAULT_MAX_BATCH_SIZE`,
		precisionType	precision = `TYPE_FASTEST`,
		deviceType	device = `DEVICE_GPU`,
		bool	allowGPUFallback = `true`,
		nvinfer1::IInt8Calibrator *	calibrator = `NULL`,
		cudaStream_t	stream = `NULL`
	)

Load a new network instance with multiple output layers.

Parameters

prototxt	File path to the deployable network prototxt
model	File path to the caffemodel
mean	File path to the mean value binary proto (NULL if none)
input_blob	The name of the input blob data to the network.
output_blobs	List of names of the output blobs from the network.
maxBatchSize	The maximum batch size that the network will be optimized for.

◆ LoadNetwork() [3/3]

bool tensorNet::LoadNetwork	(	const char *	prototxt,
		const char *	model,
		const char *	mean,
		const char *	input_blob,
		const Dims3 &	input_dims,
		const std::vector< std::string > &	output_blobs,
		uint32_t	maxBatchSize = `DEFAULT_MAX_BATCH_SIZE`,
		precisionType	precision = `TYPE_FASTEST`,
		deviceType	device = `DEVICE_GPU`,
		bool	allowGPUFallback = `true`,
		nvinfer1::IInt8Calibrator *	calibrator = `NULL`,
		cudaStream_t	stream = `NULL`
	)

Load a new network instance (this variant is used for UFF models)

Parameters

prototxt	File path to the deployable network prototxt
model	File path to the caffemodel
mean	File path to the mean value binary proto (NULL if none)
input_blob	The name of the input blob data to the network.
input_dims	The dimensions of the input blob (used for UFF).
output_blobs	List of names of the output blobs from the network.
maxBatchSize	The maximum batch size that the network will be optimized for.

◆ PrintProfilerTimes()

void tensorNet::PrintProfilerTimes ( )

inline

Print the profiler times (in millseconds).

◆ ProfileModel()

bool tensorNet::ProfileModel	(	const std::string &	deployFile,
		const std::string &	modelFile,
		const char *	input,
		const Dims3 &	inputDims,
		const std::vector< std::string > &	outputs,
		uint32_t	maxBatchSize,
		precisionType	precision,
		deviceType	device,
		bool	allowGPUFallback,
		nvinfer1::IInt8Calibrator *	calibrator,
		std::ostream &	modelStream
	)

protected

Create and output an optimized network model.

Note: this function is automatically used by LoadNetwork, but also can be used individually to perform the network operations offline.

Parameters

deployFile	name for network prototxt
modelFile	name for model
outputs	network outputs
maxBatchSize	maximum batch size
modelStream	output model stream

◆ PROFILER_BEGIN()

void tensorNet::PROFILER_BEGIN ( profilerQuery query )

inlineprotected

Begin a profiling query, before network is run.

◆ PROFILER_END()

void tensorNet::PROFILER_END ( profilerQuery query )

inlineprotected

End a profiling query, after the network is run.

◆ PROFILER_QUERY()

bool tensorNet::PROFILER_QUERY ( profilerQuery query )

inlineprotected

Query the CUDA part of a profiler query.

◆ SetStream()

void tensorNet::SetStream ( cudaStream_t stream )

Set the stream that the device is operating on.

Member Data Documentation

◆ gLogger

tensorNet::Logger tensorNet::gLogger

protected

◆ gProfiler

tensorNet::Profiler tensorNet::gProfiler

protected

◆ mAllowGPUFallback

bool tensorNet::mAllowGPUFallback

protected

◆ mCacheCalibrationPath

std::string tensorNet::mCacheCalibrationPath

protected

◆ mCacheEnginePath

std::string tensorNet::mCacheEnginePath

protected

◆ mContext

nvinfer1::IExecutionContext* tensorNet::mContext

protected

◆ mDevice

deviceType tensorNet::mDevice

protected

◆ mEnableDebug

bool tensorNet::mEnableDebug

protected

◆ mEnableProfiler

bool tensorNet::mEnableProfiler

protected

◆ mEngine

nvinfer1::ICudaEngine* tensorNet::mEngine

protected

◆ mEventsCPU

timespec tensorNet::mEventsCPU[PROFILER_TOTAL *2]

protected

◆ mEventsGPU

cudaEvent_t tensorNet::mEventsGPU[PROFILER_TOTAL *2]

protected

◆ mHeight

uint32_t tensorNet::mHeight

protected

◆ mInfer

nvinfer1::IRuntime* tensorNet::mInfer

protected

◆ mInputBlobName

std::string tensorNet::mInputBlobName

protected

◆ mInputCPU

float* tensorNet::mInputCPU

protected

◆ mInputCUDA

float* tensorNet::mInputCUDA

protected

◆ mInputDims

Dims3 tensorNet::mInputDims

protected

◆ mInputSize

uint32_t tensorNet::mInputSize

protected

◆ mMaxBatchSize

uint32_t tensorNet::mMaxBatchSize

protected

◆ mMeanPath

std::string tensorNet::mMeanPath

protected

◆ mModelPath

std::string tensorNet::mModelPath

protected

◆ mModelType

modelType tensorNet::mModelType

protected

◆ mOutputs

std::vector<outputLayer> tensorNet::mOutputs

protected

◆ mPrecision

precisionType tensorNet::mPrecision

protected

◆ mProfilerQueriesDone

uint32_t tensorNet::mProfilerQueriesDone

protected

◆ mProfilerQueriesUsed

uint32_t tensorNet::mProfilerQueriesUsed

protected

◆ mProfilerTimes

float2 tensorNet::mProfilerTimes[PROFILER_TOTAL+1]

protected

◆ mPrototxtPath

std::string tensorNet::mPrototxtPath

protected

◆ mStream

cudaStream_t tensorNet::mStream

protected

◆ mWidth

uint32_t tensorNet::mWidth

protected

The documentation for this class was generated from the following file:

jetson-inference/tensorNet.h

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Constructor & Destructor Documentation

◆ ~tensorNet()

◆ tensorNet()

Member Function Documentation

◆ AllowGPUFallback()

◆ CreateStream()

◆ DetectNativePrecision() [1/2]

◆ DetectNativePrecision() [2/2]

◆ DetectNativePrecisions()

◆ EnableDebug()

◆ EnableLayerProfiler()

◆ FindFastestPrecision()

◆ GetDevice()

◆ GetModelPath()

◆ GetModelType()

◆ GetNetworkTime()

◆ GetPrecision()

◆ GetProfilerTime() [1/2]

◆ GetProfilerTime() [2/2]

◆ GetPrototxtPath()

◆ GetStream()

◆ IsModelType()

◆ IsPrecision()

◆ LoadNetwork() [1/3]

◆ LoadNetwork() [2/3]

◆ LoadNetwork() [3/3]

◆ PrintProfilerTimes()

◆ ProfileModel()

◆ PROFILER_BEGIN()

◆ PROFILER_END()

◆ PROFILER_QUERY()

◆ SetStream()

Member Data Documentation

◆ gLogger

◆ gProfiler

◆ mAllowGPUFallback

◆ mCacheCalibrationPath

◆ mCacheEnginePath

◆ mContext

◆ mDevice

◆ mEnableDebug

◆ mEnableProfiler

◆ mEngine

◆ mEventsCPU

◆ mEventsGPU

◆ mHeight

◆ mInfer

◆ mInputBlobName

◆ mInputCPU

◆ mInputCUDA

◆ mInputDims

◆ mInputSize

◆ mMaxBatchSize

◆ mMeanPath

◆ mModelPath

◆ mModelType

◆ mOutputs

◆ mPrecision

◆ mProfilerQueriesDone

◆ mProfilerQueriesUsed

◆ mProfilerTimes

◆ mPrototxtPath

◆ mStream

◆ mWidth