#include <InstanceSegmentationCamera.h>

Inheritance diagram for AInstanceSegmentationCamera:

[legend]

Collaboration diagram for AInstanceSegmentationCamera:

[legend]

Public Member Functions
	AInstanceSegmentationCamera (const FObjectInitializer &ObjectInitializer)

void	Init (FCameraBaseParameters parameters, bool SimulateSensor=true) override

virtual ESensorTypes	GetSensorType () const override

Public Member Functions inherited from ACamera
	ACamera (const FObjectInitializer &ObjectInitializer)

virtual void	Init (FCameraBaseParameters parameters, bool SimulateSensor=true)

virtual ESensorTypes	GetSensorType () const override

void	ChangeCameraParameters (FCameraBaseParameters newParameters)

FCameraBaseParameters	GetCameraParameters ()

void	AddPostProcessingMaterial (const FString &Path, float Weight=1.0f)

void	RemovePostProcessingMaterial (UMaterial *Material)

USceneCaptureComponent2D *	GetCaptureComponent2D () const

UTextureRenderTarget2D *	GetCaptureRenderTarget2D () const

void	AddWidgetToWindow (UWidget *WidgetToAdd)

void	RemoveWidgetFromWindow (UWidget *WidgetToRemove)

void	SetShadowRendering (bool RenderShadows)

void	DisableShowFlags ()

void	SetTemporalAA (bool SetTemporal)

void	ResizeCamera (int32 Width=1280, int32 Height=720)

void	AddHiddenActor (AActor *Actor)

void	SaveCurrentFrameToDisk ()

int32	GetCameraWidth () const

int32	GetCameraHeight () const

void	SetUseGimbal (bool UseGimbal)

virtual FString	GetParametersAsString () const override

Public Member Functions inherited from ASensor
	ASensor (const FObjectInitializer &ObjectInitializer)

FString	ExportToJsonFile (const FString &FileName)

virtual ESensorTypes	GetSensorType () const

FString	GetSensorIdentifier () const

void	SetSensorIdentifier (const FString newIdentifier)

FString	GetSensorName () const

virtual FString	GetParametersAsString () const

AVehicle *	IsAttachedToVehicle () const

void	SetSensorName (const FString newName)

FString	GetTopicName ()

UTopic *	GetROSTopic () const

void	SetSimulateSensor (bool SimulateSensor)

bool	CanSimulateSensor () const

ASensorModel *	GetSensorModel () const

void	SetSensorModel (ASensorModel *NewSensorModel)

FORCEINLINE bool	IsROSConnected () const

UROSIntegrationGameInstance *	GetROSGameInstance () const

virtual FString	GetActorID_Implementation () const override

virtual FString	GetActorName_Implementation () const override

virtual FString	GetActorInformation_Implementation () const override

virtual void	SetActorName_Implementation (const FString &NewActorName) override

virtual void	SetActorIDAndName_Implementation (const FString &NewActorName, const FString &NewID) override

void	SetParentActorPtr (AActor *ParentActorPtr)

Public Member Functions inherited from IActorInformation
FString	GetActorID () const

FString	GetActorName () const

FString	GetActorInformation () const

void	SetActorName (const FString &NewActorName)

void	SetActorIDAndName (const FString &NewActorName, const FString &NewID)

Protected Member Functions
virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) final override

Protected Member Functions inherited from ACamera
virtual void	BeginPlay () override

virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) override

void	PreActorTick (UWorld *World, ELevelTick TickType, float DeltaSeconds)

void	UpdateCameraStabilization ()

void	EndOfFrame (UWorld *World, ELevelTick TickType, float DeltaSeconds)

void	PixelReadEndOfFrame (UWorld *World, ELevelTick TickType, float DeltaSeconds)

void	EndOfFrameParellel (float DeltaTime)

virtual void	AddProcessingToFrameBuffer (TArray< FColor > &buffer)

void	ApplyGammaCorrectionLUT (TArray< FColor > &Buffer)

void	SendImageDataToROS (const TArray< FColor > &FrameBuffer, int32 Width, int32 Height)

void	HidePrimitiveComponent (UPrimitiveComponent *PrimitiveComponent)

void	SaveImageToDisk (const TArray< FColor > FrameBuffer, int32 Width, int32 Height)

void	SaveCameraMetaDataToDisk (const FString &ImageName)

void	CreateLogFile () override

Protected Member Functions inherited from ASensor
virtual void	BeginPlay () override

virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) override

FString	CreateTimeStampString () const

virtual void	CreateROSTopic ()

virtual void	DestroyROSTopic ()

virtual void	CreateDataSavePath ()

bool	IsLogFileCreated ()

virtual void	CreateLogFile ()

void	WriteToLogFile (const FString &Message)

Additional Inherited Members
Static Public Member Functions inherited from ASensor
static void	HideComponentForAllCameras (UPrimitiveComponent *PrimitiveComponent)

static TMap< FString, FColor >	GetSemanticColors ()

static TArray< TWeakObjectPtr< UPrimitiveComponent > >	GetComponentsToHide ()

Static Public Member Functions inherited from IActorInformation
static void	SetAndValidateActorIDAndName (FString &ActorName, FString &ActorID, TWeakObjectPtr< AActor > Actor)

static bool	DestroyActorByID (const FString &ID)

static AActor *	GetActorByID (const FString &ID)

template<typename T >
static TArray< T * >	GetActorsWithInterface ()

static void	PrintAllIds ()

Public Attributes inherited from ACamera
TEnumAsByte< ETextureRenderTargetFormat >	TextureFormat = ETextureRenderTargetFormat::RTF_RGBA8

FCameraDelegate_OnWindowClosed	OnCameraWindowClosed

FCameraDelegate_OnWindowResized	OnCameraWindowResized

Public Attributes inherited from ASensor
FSensorDestroy	OnSensorDestroy

FString	AttachedToComponent

FName	AttachedToBone

Static Protected Member Functions inherited from ASensor
template<typename InStructType >
static FString	StructToString (const InStructType &InStruct)

Protected Attributes inherited from ACamera
bool	UseParallelLateTick = true

FDelegateHandle	OnPreTickDelegate

FDelegateHandle	OnPostTickDelegate

FString	CameraName = "Camera "

FString	FilePrefix = "Data/Camera_"

UTextureRenderTarget2D *	CaptureRenderTarget = nullptr

USceneCaptureComponent2D *	CaptureComponent2D = nullptr

UTexture2D *	CaptureFrameTexture

TSharedPtr< FUnrealWindow >	UnrealWindow

FTickEntry	TickEntry

TWeakObjectPtr< UMaterialInstanceDynamic >	IceMaterialInstance

TWeakObjectPtr< UMaterial >	PhysicLensDistortion

TSharedPtr< ROSMessages::sensor_msgs::Image >	ImageMsg

bool	UseReadPixels = false

APIDDrone *	DronePtr = nullptr

FRotator	SensorSetRotation

bool	AllowGimbal = true

Protected Attributes inherited from ASensor
UTopic *	ROSTopic = nullptr

bool	SendDataToROS = true

ULogFile *	LogFile = nullptr

AActor *	ParentActor = nullptr

FString	FileSavePath

UROSIntegrationGameInstance *	ROSInstance = nullptr

Static Protected Attributes inherited from ASensor
static FPrimitiveAdded	OnPrimitiveAdded

static const FName	NiagaraPointsInt = "User.PointCount"

static const FName	NiagaraHitPoints = "User.HitPoints"

static const FName	NiagaraHitColors = "User.HitColors"

static const FName	NiagaraPointsFloat = "User.Test"

Detailed Description

Class for Instance Segmentation Camera. Limitiations: 1) Unlike other sensors, this sensor only works on our fork of Unreal Engine as custom render pass is required 2) Doesn't support non-nanite meshes and sees through them (simulator maps don't have that many non-nanite meshes though) 3) Doesn't support ISMs (instance static meshes):

meaning all ISM instances have the same color and ID,
so this sensor should only be used in a map where each actor is individual mesh like Vindeln_Dev or RovaniemiForest_Dev.
See also
ACamera for how camera pixels are rendered, read, saved and sent to ROS.

Definition at line 23 of file InstanceSegmentationCamera.h.

Constructor & Destructor Documentation

◆ AInstanceSegmentationCamera()

AInstanceSegmentationCamera::AInstanceSegmentationCamera ( const FObjectInitializer & ObjectInitializer )

Definition at line 14 of file InstanceSegmentationCamera.cpp.

                                                                                                    : Super(ObjectInitializer)
{
    // Use USceneCaptureComponent2D ReadPixels method to get pixels instead of FrameGrabber API.
    // Otherwise ID's of instance segmentation are not correct
    // due to sRGB conversion done by FrameGrabber.
    UseParallelLateTick = false;
    UseReadPixels = true;
    PrimaryActorTick.bCanEverTick = false;
}

References ACamera::UseParallelLateTick, and ACamera::UseReadPixels.

Member Function Documentation

◆ EndPlay()

void AInstanceSegmentationCamera::EndPlay ( const EEndPlayReason::Type EndPlayReason )

finaloverrideprotectedvirtual

Reimplemented from ACamera.

Definition at line 52 of file InstanceSegmentationCamera.cpp.

{
    Super::EndPlay(EndPlayReason);
 
#ifdef InstanceSegmentationPass_EXISTS
    --InstanceSegmentationCameraCount;
    if (InstanceSegmentationCameraCount == 0) 
    {
        UWorld* World = GetWorld();
        if (GEngine && World)
        {
            GEngine->Exec(World, TEXT("r.EnableSegmentationRendering 0"));
        }
    }
#endif
}

References InstanceSegmentationCameraCount.

◆ GetSensorType()

virtual ESensorTypes AInstanceSegmentationCamera::GetSensorType ( ) const

inlineoverridevirtual

Get type of the sensor

Returns: Sensor's type

Reimplemented from ACamera.

Definition at line 37 of file InstanceSegmentationCamera.h.

    { 
        return ESensorTypes::InstanceSegmentationCamera;
    }

References InstanceSegmentationCamera.

◆ Init()

void AInstanceSegmentationCamera::Init	(	FCameraBaseParameters	parameters,
		bool	SimulateSensor = `true`
	)

overridevirtual

Initialize new Camera sensor

Parameters

FCameraBaseParameters CameraParameters struct

Reimplemented from ACamera.

Definition at line 24 of file InstanceSegmentationCamera.cpp.

{
    // Defined in Camera.h
    CameraName = "InstanceSegmentationCamera ";
    FilePrefix = "Data/InstanceSegmentationCamera_";
 
    AddPostProcessingMaterial("/Game/Agrarsense/Materials/PostProcessingMaterials/M_InstanceSegmentation.M_InstanceSegmentation");
    Super::Init(parameters, SimulateSensor);
 
    SetShadowRendering(false);
    DisableShowFlags();
    SetTemporalAA(false);
 
#ifdef InstanceSegmentationPass_EXISTS
    // If user is using AGRARSENSE Unreal Engine fork, we can utilize this sensor
    UWorld* World = GetWorld();
    if (GEngine && World)
    {
        GEngine->Exec(World, TEXT("r.EnableSegmentationRendering 1"));
    }
    ++InstanceSegmentationCameraCount;
#else
    // else log and destroy this Actor as the user cannot use this sensor anyway.
    SimulatorLog::Log("InstanceSegmentationCamera.cpp: This sensor is only supported with our AGRARSENSE fork of the engine.");
    Destroy();
#endif
}

References ACamera::AddPostProcessingMaterial(), ACamera::CameraName, Destroy, ACamera::DisableShowFlags(), ACamera::FilePrefix, InstanceSegmentationCameraCount, SimulatorLog::Log(), ACamera::SetShadowRendering(), and ACamera::SetTemporalAA().

Referenced by USensorFactory::SpawnInstanceSegmentationCamera().

The documentation for this class was generated from the following files:

Unreal/Agrarsense/Source/Agrarsense/Sensor/Camera/InstanceSegmentationCamera.h
Unreal/Agrarsense/Source/Agrarsense/Sensor/Camera/InstanceSegmentationCamera.cpp

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ AInstanceSegmentationCamera()

Member Function Documentation

◆ EndPlay()

◆ GetSensorType()

◆ Init()