Image segmentation with FCN-Alexnet or custom models, using TensorRT. More...

#include <segNet.h>

Inheritance diagram for segNet:

Public Types
enum	NetworkType { FCN_ALEXNET_PASCAL_VOC, FCN_ALEXNET_SYNTHIA_CVPR16, FCN_ALEXNET_SYNTHIA_SUMMER_HD, FCN_ALEXNET_SYNTHIA_SUMMER_SD, FCN_ALEXNET_CITYSCAPES_HD, FCN_ALEXNET_CITYSCAPES_SD, FCN_ALEXNET_AERIAL_FPV_720p, SEGNET_CUSTOM }
	Enumeration of pretrained/built-in network models. More...

enum	FilterMode { FILTER_POINT, FILTER_LINEAR }
	Enumeration of mask/overlay filtering modes. More...

Public Member Functions
virtual	~segNet ()
	Destroy. More...

bool	Process (float input, uint32_t width, uint32_t height, const char ignore_class="void")
	Perform the initial inferencing processing portion of the segmentation. More...

bool	Mask (uint8_t *output, uint32_t width, uint32_t height)
	Produce a grayscale binary segmentation mask, where the pixel values correspond to the class ID of the corresponding class type. More...

bool	Mask (float *output, uint32_t width, uint32_t height, FilterMode filter=FILTER_LINEAR)
	Produce a colorized RGBA segmentation mask. More...

bool	Overlay (float *output, uint32_t width, uint32_t height, FilterMode filter=FILTER_LINEAR)
	Produce the segmentation overlay alpha blended on top of the original image. More...

int	FindClassID (const char *label_name)
	Find the ID of a particular class (by label name). More...

uint32_t	GetNumClasses () const
	Retrieve the number of object classes supported in the detector. More...

const char *	GetClassLabel (uint32_t id) const
	Retrieve the description of a particular class. More...

float *	GetClassColor (uint32_t id) const
	Retrieve the class synset category of a particular class. More...

void	SetClassColor (uint32_t classIndex, float r, float g, float b, float a=255.0f)
	Set the visualization color of a particular class of object. More...

void	SetGlobalAlpha (float alpha, bool explicit_exempt=true)
	Set a global alpha value for all classes (between 0-255), (optionally except for those that have been explicitly set). More...

const char *	GetClassPath () const
	Retrieve the path to the file containing the class label descriptions. More...

uint32_t	GetGridWidth () const
	Retrieve the number of columns in the classification grid. More...

uint32_t	GetGridHeight () const
	Retrieve the number of rows in the classification grid. More...

NetworkType	GetNetworkType () const
	Retrieve the network type (alexnet or googlenet) More...

const char *	GetNetworkName () const
	Retrieve a string describing the network name. More...

Public Member Functions inherited from tensorNet
virtual	~tensorNet ()
	Destory. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean=NULL, const char input_blob="data", const char output_blob="prob", uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean, const char input_blob, const std::vector< std::string > &output_blobs, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator *calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance with multiple output layers. More...

bool	LoadNetwork (const char prototxt, const char model, const char mean, const char input_blob, const Dims3 &input_dims, const std::vector< std::string > &output_blobs, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true, nvinfer1::IInt8Calibrator *calibrator=NULL, cudaStream_t stream=NULL)
	Load a new network instance (this variant is used for UFF models) More...

void	EnableLayerProfiler ()
	Manually enable layer profiling times. More...

void	EnableDebug ()
	Manually enable debug messages and synchronization. More...

bool	AllowGPUFallback () const
	Return true if GPU fallback is enabled. More...

deviceType	GetDevice () const
	Retrieve the device being used for execution. More...

precisionType	GetPrecision () const
	Retrieve the type of precision being used. More...

bool	IsPrecision (precisionType type) const
	Check if a particular precision is being used. More...

cudaStream_t	GetStream () const
	Retrieve the stream that the device is operating on. More...

cudaStream_t	CreateStream (bool nonBlocking=true)
	Create and use a new stream for execution. More...

void	SetStream (cudaStream_t stream)
	Set the stream that the device is operating on. More...

const char *	GetPrototxtPath () const
	Retrieve the path to the network prototxt file. More...

const char *	GetModelPath () const
	Retrieve the path to the network model file. More...

modelType	GetModelType () const
	Retrieve the format of the network model. More...

bool	IsModelType (modelType type) const
	Return true if the model is of the specified format. More...

float	GetNetworkTime ()
	Retrieve the network runtime (in milliseconds). More...

float2	GetProfilerTime (profilerQuery query)
	Retrieve the profiler runtime (in milliseconds). More...

float	GetProfilerTime (profilerQuery query, profilerDevice device)
	Retrieve the profiler runtime (in milliseconds). More...

void	PrintProfilerTimes ()
	Print the profiler times (in millseconds). More...

Static Public Member Functions
static NetworkType	NetworkTypeFromStr (const char *model_name)
	Parse a string from one of the built-in pretrained models. More...

static FilterMode	FilterModeFromStr (const char *str, FilterMode default_value=FILTER_LINEAR)
	Parse a string from one of the FilterMode values. More...

static segNet *	Create (NetworkType networkType=FCN_ALEXNET_CITYSCAPES_SD, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true)
	Load a new network instance. More...

static segNet *	Create (const char prototxt_path, const char model_path, const char class_labels, const char class_colors=NULL, const char input=SEGNET_DEFAULT_INPUT, const char output=SEGNET_DEFAULT_OUTPUT, uint32_t maxBatchSize=DEFAULT_MAX_BATCH_SIZE, precisionType precision=TYPE_FASTEST, deviceType device=DEVICE_GPU, bool allowGPUFallback=true)
	Load a new network instance. More...

static segNet *	Create (int argc, char **argv)
	Load a new network instance by parsing the command line. More...

Static Public Member Functions inherited from tensorNet
static precisionType	FindFastestPrecision (deviceType device=DEVICE_GPU, bool allowInt8=true)
	Determine the fastest native precision on a device. More...

static std::vector< precisionType >	DetectNativePrecisions (deviceType device=DEVICE_GPU)
	Detect the precisions supported natively on a device. More...

static bool	DetectNativePrecision (const std::vector< precisionType > &nativeTypes, precisionType type)
	Detect if a particular precision is supported natively. More...

static bool	DetectNativePrecision (precisionType precision, deviceType device=DEVICE_GPU)
	Detect if a particular precision is supported natively. More...

Protected Member Functions
	segNet ()

bool	classify (const char *ignore_class)

bool	overlayPoint (float input, uint32_t in_width, uint32_t in_height, float output, uint32_t out_width, uint32_t out_height, bool mask_only)

bool	overlayLinear (float input, uint32_t in_width, uint32_t in_height, float output, uint32_t out_width, uint32_t out_height, bool mask_only)

bool	loadClassColors (const char *filename)

bool	loadClassLabels (const char *filename)

Protected Member Functions inherited from tensorNet
	tensorNet ()
	Constructor. More...

bool	ProfileModel (const std::string &deployFile, const std::string &modelFile, const char input, const Dims3 &inputDims, const std::vector< std::string > &outputs, uint32_t maxBatchSize, precisionType precision, deviceType device, bool allowGPUFallback, nvinfer1::IInt8Calibrator calibrator, std::ostream &modelStream)
	Create and output an optimized network model. More...

void	PROFILER_BEGIN (profilerQuery query)
	Begin a profiling query, before network is run. More...

void	PROFILER_END (profilerQuery query)
	End a profiling query, after the network is run. More...

bool	PROFILER_QUERY (profilerQuery query)
	Query the CUDA part of a profiler query. More...

Protected Attributes
std::vector< std::string >	mClassLabels

std::string	mClassPath

float *	mClassColors [2]
	array of overlay colors in shared CPU/GPU memory More...

uint8_t *	mClassMap [2]
	runtime buffer for the argmax-classified class index of each tile More...

float *	mLastInputImg
	last input image to be processed, stored for overlay More...

uint32_t	mLastInputWidth
	width in pixels of last input image to be processed More...

uint32_t	mLastInputHeight
	height in pixels of last input image to be processed More...

NetworkType	mNetworkType
	Pretrained built-in model type enumeration. More...

Protected Attributes inherited from tensorNet
tensorNet::Logger	gLogger

tensorNet::Profiler	gProfiler

std::string	mPrototxtPath

std::string	mModelPath

std::string	mMeanPath

std::string	mInputBlobName

std::string	mCacheEnginePath

std::string	mCacheCalibrationPath

deviceType	mDevice

precisionType	mPrecision

modelType	mModelType

cudaStream_t	mStream

cudaEvent_t	mEventsGPU [PROFILER_TOTAL *2]

timespec	mEventsCPU [PROFILER_TOTAL *2]

nvinfer1::IRuntime *	mInfer

nvinfer1::ICudaEngine *	mEngine

nvinfer1::IExecutionContext *	mContext

uint32_t	mWidth

uint32_t	mHeight

uint32_t	mInputSize

float *	mInputCPU

float *	mInputCUDA

float2	mProfilerTimes [PROFILER_TOTAL+1]

uint32_t	mProfilerQueriesUsed

uint32_t	mProfilerQueriesDone

uint32_t	mMaxBatchSize

bool	mEnableProfiler

bool	mEnableDebug

bool	mAllowGPUFallback

Dims3	mInputDims

std::vector< outputLayer >	mOutputs

Detailed Description

Image segmentation with FCN-Alexnet or custom models, using TensorRT.

Member Enumeration Documentation

◆ FilterMode

enum segNet::FilterMode

Enumeration of mask/overlay filtering modes.

Enumerator
FILTER_POINT	Nearest point sampling.
FILTER_LINEAR	Bilinear filtering.

◆ NetworkType

enum segNet::NetworkType

Enumeration of pretrained/built-in network models.

Enumerator
FCN_ALEXNET_PASCAL_VOC	FCN-Alexnet trained on Pascal VOC dataset.
FCN_ALEXNET_SYNTHIA_CVPR16	FCN-Alexnet trained on SYNTHIA CVPR16 dataset. Note To save disk space, this model isn't downloaded by default. Enable it in CMakePreBuild.sh
FCN_ALEXNET_SYNTHIA_SUMMER_HD	FCN-Alexnet trained on SYNTHIA SEQS summer datasets. Note To save disk space, this model isn't downloaded by default. Enable it in CMakePreBuild.sh
FCN_ALEXNET_SYNTHIA_SUMMER_SD	FCN-Alexnet trained on SYNTHIA SEQS summer datasets. Note To save disk space, this model isn't downloaded by default. Enable it in CMakePreBuild.sh
FCN_ALEXNET_CITYSCAPES_HD	FCN-Alexnet trained on Cityscapes dataset with 21 classes.
FCN_ALEXNET_CITYSCAPES_SD	FCN-Alexnet trained on Cityscapes dataset with 21 classes. Note To save disk space, this model isn't downloaded by default. Enable it in CMakePreBuild.sh
FCN_ALEXNET_AERIAL_FPV_720p	FCN-Alexnet trained on aerial first-person view of the horizon line for drones, 1280x720 and 21 output classes.
SEGNET_CUSTOM

Constructor & Destructor Documentation

◆ ~segNet()

virtual segNet::~segNet ( )

virtual

Destroy.

◆ segNet()

segNet::segNet ( )

protected

Member Function Documentation

◆ classify()

bool segNet::classify ( const char * ignore_class )

protected

◆ Create() [1/3]

static segNet* segNet::Create	(	NetworkType	networkType = `FCN_ALEXNET_CITYSCAPES_SD`,
		uint32_t	maxBatchSize = `DEFAULT_MAX_BATCH_SIZE`,
		precisionType	precision = `TYPE_FASTEST`,
		deviceType	device = `DEVICE_GPU`,
		bool	allowGPUFallback = `true`
	)

static

Load a new network instance.

◆ Create() [2/3]

static segNet* segNet::Create	(	const char *	prototxt_path,
		const char *	model_path,
		const char *	class_labels,
		const char *	class_colors = `NULL`,
		const char *	input = `SEGNET_DEFAULT_INPUT`,
		const char *	output = `SEGNET_DEFAULT_OUTPUT`,
		uint32_t	maxBatchSize = `DEFAULT_MAX_BATCH_SIZE`,
		precisionType	precision = `TYPE_FASTEST`,
		deviceType	device = `DEVICE_GPU`,
		bool	allowGPUFallback = `true`
	)

static

Load a new network instance.

Parameters

prototxt_path	File path to the deployable network prototxt
model_path	File path to the caffemodel
class_labels	File path to list of class name labels
class_colors	File path to list of class colors
input	Name of the input layer blob.

See also: SEGNET_DEFAULT_INPUT

Parameters

output Name of the output layer blob.

See also: SEGNET_DEFAULT_OUTPUT

Parameters

maxBatchSize The maximum batch size that the network will support and be optimized for.

◆ Create() [3/3]

static segNet* segNet::Create	(	int	argc,
		char **	argv
	)

static

Load a new network instance by parsing the command line.

◆ FilterModeFromStr()

static FilterMode segNet::FilterModeFromStr	(	const char *	str,
		FilterMode	default_value = `FILTER_LINEAR`
	)

static

Parse a string from one of the FilterMode values.

Valid strings are "point", and "linear"

Returns: one of the segNet::FilterMode enums, or default segNet::FILTER_LINEAR on an error.

◆ FindClassID()

int segNet::FindClassID ( const char * label_name )

Find the ID of a particular class (by label name).

◆ GetClassColor()

float* segNet::GetClassColor ( uint32_t id ) const

inline

Retrieve the class synset category of a particular class.

◆ GetClassLabel()

const char* segNet::GetClassLabel ( uint32_t id ) const

inline

Retrieve the description of a particular class.

◆ GetClassPath()

const char* segNet::GetClassPath ( ) const

inline

Retrieve the path to the file containing the class label descriptions.

◆ GetGridHeight()

uint32_t segNet::GetGridHeight ( ) const

inline

Retrieve the number of rows in the classification grid.

This indicates the resolution of the raw segmentation output.

◆ GetGridWidth()

uint32_t segNet::GetGridWidth ( ) const

inline

Retrieve the number of columns in the classification grid.

This indicates the resolution of the raw segmentation output.

◆ GetNetworkName()

const char* segNet::GetNetworkName ( ) const

inline

Retrieve a string describing the network name.

◆ GetNetworkType()

NetworkType segNet::GetNetworkType ( ) const

inline

Retrieve the network type (alexnet or googlenet)

◆ GetNumClasses()

uint32_t segNet::GetNumClasses ( ) const

inline

Retrieve the number of object classes supported in the detector.

◆ loadClassColors()

bool segNet::loadClassColors ( const char * filename )

protected

◆ loadClassLabels()

bool segNet::loadClassLabels ( const char * filename )

protected

◆ Mask() [1/2]

bool segNet::Mask	(	uint8_t *	output,
		uint32_t	width,
		uint32_t	height
	)

Produce a grayscale binary segmentation mask, where the pixel values correspond to the class ID of the corresponding class type.

◆ Mask() [2/2]

bool segNet::Mask	(	float *	output,
		uint32_t	width,
		uint32_t	height,
		FilterMode	filter = `FILTER_LINEAR`
	)

Produce a colorized RGBA segmentation mask.

◆ NetworkTypeFromStr()

static NetworkType segNet::NetworkTypeFromStr ( const char * model_name )

static

Parse a string from one of the built-in pretrained models.

Valid names are "cityscapes-hd", "cityscapes-sd", "pascal-voc", ect.

Returns: one of the segNet::NetworkType enums, or segNet::CUSTOM on invalid string.

◆ Overlay()

bool segNet::Overlay	(	float *	output,
		uint32_t	width,
		uint32_t	height,
		FilterMode	filter = `FILTER_LINEAR`
	)

Produce the segmentation overlay alpha blended on top of the original image.

Parameters

input	float4 input image in CUDA device memory, RGBA colorspace with values 0-255.
output	float4 output image in CUDA device memory, RGBA colorspace with values 0-255.
width	width of the input image in pixels.
height	height of the input image in pixels.
ignore_class	label name of class to ignore in the classification (or NULL to process all).
type	overlay visualization options

Returns: true on success, false on error.

◆ overlayLinear()

bool segNet::overlayLinear	(	float *	input,
		uint32_t	in_width,
		uint32_t	in_height,
		float *	output,
		uint32_t	out_width,
		uint32_t	out_height,
		bool	mask_only
	)

protected

◆ overlayPoint()

bool segNet::overlayPoint	(	float *	input,
		uint32_t	in_width,
		uint32_t	in_height,
		float *	output,
		uint32_t	out_width,
		uint32_t	out_height,
		bool	mask_only
	)

protected

◆ Process()

bool segNet::Process	(	float *	input,
		uint32_t	width,
		uint32_t	height,
		const char *	ignore_class = `"void"`
	)

Perform the initial inferencing processing portion of the segmentation.

The results can then be visualized using the Overlay() and Mask() functions.

Parameters

input	float4 input image in CUDA device memory, RGBA colorspace with values 0-255.
width	width of the input image in pixels.
height	height of the input image in pixels.
ignore_class	label name of class to ignore in the classification (or NULL to process all).

◆ SetClassColor()

void segNet::SetClassColor	(	uint32_t	classIndex,
		float	r,
		float	g,
		float	b,
		float	a = `255.0f`
	)

Set the visualization color of a particular class of object.

◆ SetGlobalAlpha()

void segNet::SetGlobalAlpha	(	float	alpha,
		bool	explicit_exempt = `true`
	)

Set a global alpha value for all classes (between 0-255), (optionally except for those that have been explicitly set).

Member Data Documentation

◆ mClassColors

float* segNet::mClassColors[2]

protected

array of overlay colors in shared CPU/GPU memory

◆ mClassLabels

std::vector<std::string> segNet::mClassLabels

protected

◆ mClassMap

uint8_t* segNet::mClassMap[2]

protected

runtime buffer for the argmax-classified class index of each tile

◆ mClassPath

std::string segNet::mClassPath

protected

◆ mLastInputHeight

uint32_t segNet::mLastInputHeight

protected

height in pixels of last input image to be processed

◆ mLastInputImg

float* segNet::mLastInputImg

protected

last input image to be processed, stored for overlay

◆ mLastInputWidth

uint32_t segNet::mLastInputWidth

protected

width in pixels of last input image to be processed

◆ mNetworkType

NetworkType segNet::mNetworkType

protected

Pretrained built-in model type enumeration.

The documentation for this class was generated from the following file:

jetson-inference/segNet.h

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Detailed Description

Member Enumeration Documentation

◆ FilterMode

◆ NetworkType

Constructor & Destructor Documentation

◆ ~segNet()

◆ segNet()

Member Function Documentation

◆ classify()

◆ Create() [1/3]

◆ Create() [2/3]

◆ Create() [3/3]

◆ FilterModeFromStr()

◆ FindClassID()

◆ GetClassColor()

◆ GetClassLabel()

◆ GetClassPath()

◆ GetGridHeight()

◆ GetGridWidth()

◆ GetNetworkName()

◆ GetNetworkType()

◆ GetNumClasses()

◆ loadClassColors()

◆ loadClassLabels()

◆ Mask() [1/2]

◆ Mask() [2/2]

◆ NetworkTypeFromStr()

◆ Overlay()

◆ overlayLinear()

◆ overlayPoint()

◆ Process()

◆ SetClassColor()

◆ SetGlobalAlpha()

Member Data Documentation

◆ mClassColors

◆ mClassLabels

◆ mClassMap

◆ mClassPath

◆ mLastInputHeight

◆ mLastInputImg

◆ mLastInputWidth

◆ mNetworkType