#include <DVSCamera.h>

Inheritance diagram for ADVSCamera:

[legend]

Collaboration diagram for ADVSCamera:

[legend]

Public Member Functions
	ADVSCamera (const FObjectInitializer &ObjectInitializer)

virtual void	AddProcessingToFrameBuffer (TArray< FColor > &buffer) override

virtual ESensorTypes	GetSensorType () const override

void	ChangeDVSCameraParameters (FDVSCameraParameters NewParameters)

FDVSCameraParameters	GetDVSCameraParameters () const

UTopic *	GetDVSTopic () const

FString	GetDVSTopicName () const

virtual FString	GetParametersAsString () const override

Public Member Functions inherited from ACamera
	ACamera (const FObjectInitializer &ObjectInitializer)

virtual void	Init (FCameraBaseParameters parameters, bool SimulateSensor=true)

virtual ESensorTypes	GetSensorType () const override

void	ChangeCameraParameters (FCameraBaseParameters newParameters)

FCameraBaseParameters	GetCameraParameters ()

void	AddPostProcessingMaterial (const FString &Path, float Weight=1.0f)

void	RemovePostProcessingMaterial (UMaterial *Material)

USceneCaptureComponent2D *	GetCaptureComponent2D () const

UTextureRenderTarget2D *	GetCaptureRenderTarget2D () const

void	AddWidgetToWindow (UWidget *WidgetToAdd)

void	RemoveWidgetFromWindow (UWidget *WidgetToRemove)

void	SetShadowRendering (bool RenderShadows)

void	SetTemporalAA (bool SetTemporal)

void	ResizeCamera (int Width=1280, int Height=720)

void	SaveCurrentFrameToDisk ()

int	GetCameraWidth () const

int	GetCameraHeight () const

virtual FString	GetParametersAsString () const override

Public Member Functions inherited from ASensor
	ASensor (const FObjectInitializer &ObjectInitializer)

FString	ExportToJsonFile (const FString &FileName)

virtual ESensorTypes	GetSensorType () const

FString	GetSensorIdentifier () const

void	SetSensorIdentifier (const FString newIdentifier)

FString	GetSensorName () const

virtual FString	GetParametersAsString () const

void	SetSensorName (const FString newName)

virtual FString	GetTopicName ()

UTopic *	GetROSTopic () const

void	SetSimulateSensor (bool SimulateSensor)

bool	CanSimulateSensor () const

ASensorModel *	GetSensorModel () const

void	SetSensorModel (ASensorModel *NewSensorModel)

bool	IsROSConnected () const

UROSIntegrationGameInstance *	GetROSGameInstance () const

virtual FString	GetActorID_Implementation () const override

virtual FString	GetActorName_Implementation () const override

virtual FString	GetActorInformation_Implementation () const override

virtual void	SetActorName_Implementation (const FString &NewActorName) override

virtual void	SetActorIDAndName_Implementation (const FString &NewActorName, const FString &NewID) override

Public Member Functions inherited from IActorInformation
FString	GetActorID () const

FString	GetActorName () const

FString	GetActorInformation () const

void	SetActorName (const FString &NewActorName)

void	SetActorIDAndName (const FString &NewActorName, const FString &NewID)

Protected Member Functions
void	CreateROSTopic () override

Protected Member Functions inherited from ACamera
virtual void	BeginPlay () override

virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) override

virtual void	PreActorTick (UWorld *World, ELevelTick TickType, float DeltaSeconds)

virtual void	EndOfFrame (UWorld *World, ELevelTick TickType, float DeltaSeconds)

virtual void	EndOfFrameParellel (float DeltaTime)

virtual void	AddProcessingToFrameBuffer (TArray< FColor > &buffer)

virtual void	SendImageDataToROS (const TArray< FColor > &FrameBuffer, int32 Width, int32 Height)

void	HidePrimitiveComponent (UPrimitiveComponent *PrimitiveComponent)

void	SaveImageToDisk (const TArray< FColor > FrameBuffer, int32 Width, int32 Height)

void	SaveCameraMetaDataToDisk (const FString &ImageName)

void	CreateLogFile () override

Protected Member Functions inherited from ASensor
virtual void	BeginPlay () override

virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) override

virtual void	CreateROSTopic ()

virtual void	DestroyROSTopic ()

virtual void	CreateDataSavePath ()

bool	IsLogFileCreated ()

virtual void	CreateLogFile ()

void	WriteToLogFile (const FString &Message)

Private Member Functions
void	DVSInit (FDVSCameraParameters Parameters, bool SimulateSensor=true)

void	Init (FCameraBaseParameters parameters, bool SimulateSensor=true) override

virtual void	EndPlay (const EEndPlayReason::Type EndPlayReason) override

void	OnMainCameraWindowResized (ACamera *Camera, FCameraBaseParameters Params)

void	ImageToGray (const TArray< FColor > &Buffer)

void	ImageToLogGray (const TArray< FColor > &Buffer)

void	SendRawDVSImageDataToROS ()

void	DVSWindowClosedCallback (const TSharedRef< SWindow > &Window)

void	UpdateDVSWindowOutput (const TArray< FColor > &Buffer)

float	FColorToGrayScaleFloat (const FColor &Color)

void	ChangeParametersInternal ()

std::vector< DVSEvent >	SimulateDVS (const TArray< FColor > &Buffer)

constexpr std::int64_t	SecToNanoSec (double seconds)

constexpr double	NanoSecToSecTrunc (std::int64_t nanoseconds)

Private Attributes
std::vector< DVSEvent >	EventArray

TArray< double >	LastEventTimestamp

TArray< float >	PreviousImage

TArray< float >	LastImage

TArray< float >	RefValues

std::int64_t	CurrentTime

FDVSCameraParameters	TempDVSCameraParameters

FDVSCameraParameters	DVSCameraParameters

UTopic *	DvsTopic = nullptr

TArray< FColor >	DVSOutput

bool	SetParentParameters = false

bool	ParametersChanged = false

TSharedPtr< FUnrealWindow >	DVSUnrealWindow

TSharedPtr< ROSMessages::sensor_msgs::PointCloud2 >	PointCloudMessage

FOnWindowClosed	OnDVSWindowClosed

Static Private Attributes
static TArray< int32 >	ApprovedWidths

Friends
class	USensorFactory

Additional Inherited Members
Static Public Member Functions inherited from ASensor
static void	HideComponentForAllCameras (UPrimitiveComponent *PrimitiveComponent)

static TMap< FString, FColor >	GetSemanticColors ()

static TArray< TWeakObjectPtr< UPrimitiveComponent > >	GetComponentsToHide ()

Static Public Member Functions inherited from IActorInformation
static void	SetAndValidateActorIDAndName (FString &ActorName, FString &ActorID, TWeakObjectPtr< AActor > Actor)

static bool	DestroyActorByID (const FString &ID)

static AActor *	GetActorByID (const FString &ID)

template<typename T >
static TArray< T * >	GetActorsWithInterface ()

static void	PrintAllIds ()

Public Attributes inherited from ACamera
TEnumAsByte< ETextureRenderTargetFormat >	TextureFormat = ETextureRenderTargetFormat::RTF_RGBA8

FCameraDelegate_OnWindowClosed	OnCameraWindowClosed

FCameraDelegate_OnWindowResized	OnCameraWindowResized

Public Attributes inherited from ASensor
FSensorDestroy	OnSensorDestroy

FString	AttachedToComponent

FName	AttachedToBone

Static Protected Member Functions inherited from ASensor
template<typename InStructType >
static FString	StructToString (const InStructType &InStruct)

Protected Attributes inherited from ACamera
bool	UseParallelLateTick = true

FDelegateHandle	OnPreTickDelegate

FDelegateHandle	OnPostTickDelegate

FString	CameraName = "Camera "

FString	FilePrefix = "Data/Camera_"

UTextureRenderTarget2D *	CaptureRenderTarget = nullptr

USceneCaptureComponent2D *	CaptureComponent2D = nullptr

UTexture2D *	CaptureFrameTexture

TSharedPtr< FUnrealWindow >	UnrealWindow

FTickEntry	TickEntry

TWeakObjectPtr< UMaterialInstanceDynamic >	IceMaterialInstance

TWeakObjectPtr< UMaterial >	PhysicLensDistortion

Protected Attributes inherited from ASensor
UTopic *	ROSTopic = nullptr

bool	SendDataToROS = true

ULogFile *	LogFile = nullptr

FString	FileSavePath

UROSIntegrationGameInstance *	ROSInstance = nullptr

Static Protected Attributes inherited from ASensor
static FPrimitiveAdded	OnPrimitiveAdded

static const FName	NiagaraPointsInt = "User.PointCount"

static const FName	NiagaraHitPoints = "User.HitPoints"

static const FName	NiagaraHitColors = "User.HitColors"

static const FName	NiagaraPointsFloat = "User.Test"

Detailed Description

Class for Dynamic Vision Camera.

Definition at line 22 of file DVSCamera.h.

Constructor & Destructor Documentation

◆ ADVSCamera()

ADVSCamera::ADVSCamera ( const FObjectInitializer & ObjectInitializer )

Constructor for ADVSCamera.

Parameters

ObjectInitializer - The object initializer.

Definition at line 30 of file DVSCamera.cpp.

                                                                  : Super(ObjectInitializer)
{
    // Cannot tick parallel due UTexture2D locks
    UseParallelLateTick = false;
 
    PrimaryActorTick.bCanEverTick = true;
    PrimaryActorTick.TickGroup = TG_PostUpdateWork;
}

References ACamera::UseParallelLateTick.

Member Function Documentation

◆ AddProcessingToFrameBuffer()

void ADVSCamera::AddProcessingToFrameBuffer ( TArray< FColor > & buffer )

overridevirtual

Add processing to the frame buffer.

Parameters

buffer - The frame buffer to process.

Reimplemented from ACamera.

Definition at line 415 of file DVSCamera.cpp.

{
    SimulateDVS(buffer);
 
    FGraphEventRef SendDataToROSTask = FFunctionGraphTask::CreateAndDispatchWhenReady([this]()
        {
            SendRawDVSImageDataToROS();
        }, TStatId(), nullptr, ENamedThreads::AnyBackgroundHiPriTask);
 
    // Bind DVSOutput to buffer
    buffer = DVSOutput;
 
    // Update secondary Window with the new DVS image
    UpdateDVSWindowOutput(buffer);
 
    if (SendDataToROSTask.IsValid())
    {
        FTaskGraphInterface::Get().WaitUntilTaskCompletes(SendDataToROSTask);
    }
 
    if (ParametersChanged)
    {
        ChangeParametersInternal();
    }
}

References ChangeParametersInternal(), DVSOutput, ParametersChanged, SendRawDVSImageDataToROS(), SimulateDVS(), and UpdateDVSWindowOutput().

◆ ChangeDVSCameraParameters()

void ADVSCamera::ChangeDVSCameraParameters ( FDVSCameraParameters NewParameters )

Change the DVS camera parameters.

Parameters

NewParameters - The new DVS camera parameters.

Definition at line 154 of file DVSCamera.cpp.

{
    TempDVSCameraParameters = NewParameters;
    SetParentParameters = true;
 
    if (!CanSimulateSensor())
    {
        ChangeParametersInternal();
    }
    else
    {
        ParametersChanged = true;
    }
}

References ASensor::CanSimulateSensor(), ChangeParametersInternal(), ParametersChanged, SetParentParameters, and TempDVSCameraParameters.

◆ ChangeParametersInternal()

void ADVSCamera::ChangeParametersInternal ( )

private

Internal function to change camera parameters.

Definition at line 441 of file DVSCamera.cpp.

{
    ParametersChanged = false;
    DVSCameraParameters = TempDVSCameraParameters;
 
    if (SetParentParameters)
    {
        SetParentParameters = false;
 
        // Change FCameraBaseParameters
        ChangeCameraParameters(DVSCameraParameters.CameraParameters);
    }
    else
    {
        if (DVSUnrealWindow.IsValid())
        {
            const int32 Width = GetCameraWidth();
            const int32 Height = GetCameraHeight();
            DVSUnrealWindow->ResizeWindow(Width, Height);
        }
    }
}

References FDVSCameraParameters::CameraParameters, ACamera::ChangeCameraParameters(), DVSCameraParameters, DVSUnrealWindow, ACamera::GetCameraHeight(), ACamera::GetCameraWidth(), ParametersChanged, SetParentParameters, and TempDVSCameraParameters.

Referenced by AddProcessingToFrameBuffer(), and ChangeDVSCameraParameters().

◆ CreateROSTopic()

void ADVSCamera::CreateROSTopic ( )

overrideprotectedvirtual

Initialize the camera topic.

Reimplemented from ASensor.

Definition at line 358 of file DVSCamera.cpp.

{
    if (!ROSInstance || !IsROSConnected())
    {
        return;
    }
 
    FString ID = GetSensorIdentifier();
 
    if (!DvsTopic)
    {
        FString RawDataTopicName = FString::Printf(TEXT("/agrarsense/out/sensors/%s_raw"), *ID);
 
        DvsTopic = NewObject<UTopic>(UTopic::StaticClass());
        DvsTopic->Init(ROSInstance->ROSIntegrationCore, RawDataTopicName, TEXT("sensor_msgs/PointCloud2"));
        DvsTopic->Advertise();
 
#if WITH_EDITOR
        UE_LOG(LogTemp, Warning, TEXT("DVSCamera.cpp: Created DVS Camera raw data ROS topic. Topic name: %s"), *RawDataTopicName);
#endif
    }
 
    if (!ROSTopic)
    {
        FString ImageTopicName = FString::Printf(TEXT("/agrarsense/out/sensors/%s"), *ID);
 
        ROSTopic = NewObject<UTopic>(UTopic::StaticClass());
        ROSTopic->Init(ROSInstance->ROSIntegrationCore, ImageTopicName, TEXT("sensor_msgs/Image"));
        ROSTopic->Advertise();
 
#if WITH_EDITOR
        UE_LOG(LogTemp, Warning, TEXT("DVSCamera.cpp: Created DVS Camera Image ROS topic. Topic name: %s"), *ImageTopicName);
#endif
    }
}

References DvsTopic, ASensor::GetSensorIdentifier(), ASensor::IsROSConnected(), ASensor::ROSInstance, and ASensor::ROSTopic.

◆ DVSInit()

void ADVSCamera::DVSInit	(	FDVSCameraParameters	Parameters,
		bool	SimulateSensor = `true`
	)

private

Initialize the DVS camera with the specified parameters.

Parameters

Parameters - The DVS camera parameters.

Definition at line 39 of file DVSCamera.cpp.

{
    DVSCameraParameters = Parameters;
    Init(DVSCameraParameters.CameraParameters, SimulateSensor);
}

References FDVSCameraParameters::CameraParameters, DVSCameraParameters, and Init().

Referenced by USensorFactory::SpawnDVSCamera().

◆ DVSWindowClosedCallback()

void ADVSCamera::DVSWindowClosedCallback ( const TSharedRef< SWindow > & Window )

private

Callback when the DVS window is closed.

Parameters

Window - The closed window.

Definition at line 169 of file DVSCamera.cpp.

{
#if WITH_EDITOR
    UE_LOG(LogTemp, Warning, TEXT("DVSCamera.cpp: DVS Camera output window closed."));
#endif
}

Referenced by Init().

◆ EndPlay()

void ADVSCamera::EndPlay ( const EEndPlayReason::Type EndPlayReason )

overrideprivatevirtual

Called when the camera's play ends.

Parameters

EndPlayReason - The reason for the end of play.

Reimplemented from ACamera.

Definition at line 114 of file DVSCamera.cpp.

{
    Super::EndPlay(EndPlayReason);
 
    if (DVSUnrealWindow.IsValid())
    {
        OnDVSWindowClosed.Unbind();
        DVSUnrealWindow->DestroyWindow();
        DVSUnrealWindow.Reset();
    }
 
    this->OnCameraWindowResized.RemoveDynamic(this, &ADVSCamera::OnMainCameraWindowResized);
 
    LastEventTimestamp.Empty();
    PreviousImage.Empty();
    LastImage.Empty();
    RefValues.Empty();
    DVSOutput.Empty();
 
    EventArray.clear();
 
    PointCloudMessage.Reset();
 
    if (DvsTopic)
    {
        DvsTopic->Unadvertise();
        DvsTopic->Unsubscribe();
        DvsTopic->MarkAsDisconnected();
        DvsTopic->ConditionalBeginDestroy();
        DvsTopic = nullptr;
    }
}

References DVSOutput, DvsTopic, DVSUnrealWindow, EventArray, LastEventTimestamp, LastImage, ACamera::OnCameraWindowResized, OnDVSWindowClosed, OnMainCameraWindowResized(), PointCloudMessage, PreviousImage, and RefValues.

◆ FColorToGrayScaleFloat()

float ADVSCamera::FColorToGrayScaleFloat ( const FColor & Color )

private

Convert an FColor to grayscale as a floating-point value.

Parameters

Color - The color to convert.

Returns: The grayscale value as a float.

Definition at line 176 of file DVSCamera.cpp.

{
    return R_WEIGHT * Color.R + G_WEIGHT * Color.G + B_WEIGHT * Color.B;
}

References B_WEIGHT, G_WEIGHT, and R_WEIGHT.

Referenced by ImageToLogGray().

◆ GetDVSCameraParameters()

FDVSCameraParameters ADVSCamera::GetDVSCameraParameters ( ) const

inline

Get the current DVS camera parameters.

Returns: The DVS camera parameters.

Definition at line 58 of file DVSCamera.h.

    {
        return DVSCameraParameters;
    }

Referenced by USimulatorJsonExporter::ExportSensorToJSON().

◆ GetDVSTopic()

UTopic * ADVSCamera::GetDVSTopic ( ) const

inline

Definition at line 64 of file DVSCamera.h.

    {
        return DvsTopic;
    }

◆ GetDVSTopicName()

FString ADVSCamera::GetDVSTopicName ( ) const

inline

Definition at line 70 of file DVSCamera.h.

    {
        FString Name;
        if (DvsTopic)
        {
            Name = DvsTopic->GetTopicName();
        }
        return Name;
    }

◆ GetParametersAsString()

virtual FString ADVSCamera::GetParametersAsString ( ) const

inlineoverridevirtual

Get this sensor parameters struct fields as one string.

Reimplemented from ACamera.

Definition at line 83 of file DVSCamera.h.

    {
        return StructToString(DVSCameraParameters);
    }

◆ GetSensorType()

virtual ESensorTypes ADVSCamera::GetSensorType ( ) const

inlineoverridevirtual

Get the sensor type.

Returns: The sensor type.

Reimplemented from ACamera.

Definition at line 44 of file DVSCamera.h.

44{ return ESensorTypes::DVSCamera; }

ESensorTypes::DVSCamera

@ DVSCamera

References DVSCamera.

◆ ImageToGray()

void ADVSCamera::ImageToGray ( const TArray< FColor > & Buffer )

private

Convert an image to grayscale.

Parameters

Buffer - The image buffer to convert.

Definition at line 181 of file DVSCamera.cpp.

{
    LastImage.SetNumUninitialized(Buffer.Num());
 
    // Convert image to gray image values
    ParallelFor(Buffer.Num(), [&](int32 i)
        {
            LastImage[i] = FColorToGrayScaleFloat(Buffer[i]);
        }, !DVSCameraParameters.ParalellImageConversion);
}

References DVSCameraParameters, LastImage, and FDVSCameraParameters::ParalellImageConversion.

Referenced by SimulateDVS().

◆ ImageToLogGray()

void ADVSCamera::ImageToLogGray ( const TArray< FColor > & Buffer )

private

Convert an image to log grayscale.

Parameters

Buffer - The image buffer to convert.

Definition at line 192 of file DVSCamera.cpp.

{
    const int32 Size = Buffer.Num();
    const float LogEps = DVSCameraParameters.LogEps;
 
    LastImage.SetNumUninitialized(Size);
 
    // Convert FColors to logarithmic grayscale
    ParallelFor(Size, [&](int32 Index)
        {
            float GrayScale = FColorToGrayScaleFloat(Buffer[Index]) * INV_MAX_VALUE;
            LastImage[Index] = std::log(LogEps + GrayScale);
        }, !DVSCameraParameters.ParalellImageConversion); // Force single thread? Default false.
}

References DVSCameraParameters, FColorToGrayScaleFloat(), INV_MAX_VALUE, LastImage, FDVSCameraParameters::LogEps, and FDVSCameraParameters::ParalellImageConversion.

Referenced by SimulateDVS().

◆ Init()

void ADVSCamera::Init	(	FCameraBaseParameters	parameters,
		bool	SimulateSensor = `true`
	)

overrideprivatevirtual

Initialize the camera with the specified parameters.

Parameters

parameters	- The camera parameters.
SimulateSensor	- Whether to simulate the sensor.

Reimplemented from ACamera.

Definition at line 45 of file DVSCamera.cpp.

{
    // Defined in Camera.h
    CameraName = "DVSCamera ";
    FilePrefix = "Data/DVSCamera_";
 
    Super::Init(DVSCameraParameters.CameraParameters, SimulateSensor);
 
    const int32 Width = GetCameraWidth();
    const int32 Height = GetCameraHeight();
    const int32 Size = Width * Height;
 
    auto DVSEventFill = DVSEvent();
    EventArray.resize(Size, DVSEventFill);
 
    DVSOutput.Init(FColor::Black, Size);
 
    this->OnCameraWindowResized.AddUniqueDynamic(this, &ADVSCamera::OnMainCameraWindowResized);
 
    if (!DVSCameraParameters.VisualizeDVSCamera)
    {
        return;
    }
 
    // Since the Camera.cpp sensor works by USceneCaptureComponent2D rendering into custom Window texture,
    // where its pixels are read through FrameGrabber API,
    // We are unable (or at least I couldn't figure a way) to manipulate the pixels 
    // AND visualize it in the same Window texture without flickering.
    // So here we can optionally create another window where we can visualize the DVS camera output.
    // This Window can be closed safely and it does not destroy the sensor itself, and it will not stop sending data to ROS.
 
    // Create another Window for DVS visualization.
    FString windowName = GetSensorIdentifier() + " Output";
    DVSUnrealWindow = MakeShareable(new FUnrealWindow(Width, Height, windowName));
    if (DVSUnrealWindow.IsValid())
    {
        // Only draw when requested.
        DVSUnrealWindow->SetAutomaticDraw(false);
 
        OnDVSWindowClosed.BindUObject(this, &ADVSCamera::DVSWindowClosedCallback);
        DVSUnrealWindow->GetSWindow()->SetOnWindowClosed(OnDVSWindowClosed);
        DVSUnrealWindow->ResizeWindow(Width, Height);
    }
 
#ifdef EngineAllowRenderingMinimized
    // Minimize the DVS camera window (which just shows the RGB camera image without DVS conversion),
    // so we are only left with DVS window
    if (UnrealWindow.IsValid())
    {
        UnrealWindow->GetSWindow()->Minimize();
    }
#endif
 
    // Setup PointCloudMessage
    PointCloudMessage = MakeShared<ROSMessages::sensor_msgs::PointCloud2>();
    PointCloudMessage->header.frame_id = "world";
    PointCloudMessage->height = 1;
    PointCloudMessage->is_dense = true;
 
    PointCloudMessage->fields.Reserve(4);
 
    PointCloudMessage->fields.Add({ "x", 0, ROSMessages::sensor_msgs::PointCloud2::PointField::UINT16, 1 });
    PointCloudMessage->fields.Add({ "y", 2, ROSMessages::sensor_msgs::PointCloud2::PointField::UINT16, 1 });
    PointCloudMessage->fields.Add({ "t", 4, ROSMessages::sensor_msgs::PointCloud2::PointField::UINT32, 1 });
    PointCloudMessage->fields.Add({ "pol", 8, ROSMessages::sensor_msgs::PointCloud2::PointField::UINT8, 1 });
 
    PointCloudMessage->point_step = 9;
}

References ACamera::CameraName, FDVSCameraParameters::CameraParameters, DVSCameraParameters, DVSOutput, DVSUnrealWindow, DVSWindowClosedCallback(), EventArray, ACamera::FilePrefix, ACamera::GetCameraHeight(), ACamera::GetCameraWidth(), ASensor::GetSensorIdentifier(), ACamera::OnCameraWindowResized, OnDVSWindowClosed, OnMainCameraWindowResized(), PointCloudMessage, ACamera::UnrealWindow, and FDVSCameraParameters::VisualizeDVSCamera.

Referenced by DVSInit().

◆ NanoSecToSecTrunc()

constexpr double ADVSCamera::NanoSecToSecTrunc ( std::int64_t nanoseconds )

inlineconstexprprivate

Convert nanoseconds to seconds with truncation.

Parameters

nanoseconds - The time in nanoseconds to convert.

Returns: The time in seconds as a double.

Definition at line 227 of file DVSCamera.h.

    {
        return static_cast<double>(nanoseconds) / 1e9;
    }

Referenced by SimulateDVS().

◆ OnMainCameraWindowResized()

void ADVSCamera::OnMainCameraWindowResized	(	ACamera *	Camera,
		FCameraBaseParameters	Params
	)

private

Callback when the main camera window is resized.

Parameters

Camera	- The camera.
Params	- The camera parameters.

Definition at line 147 of file DVSCamera.cpp.

{
    TempDVSCameraParameters = DVSCameraParameters;
    SetParentParameters = false;
    ParametersChanged = true;
}

References DVSCameraParameters, ParametersChanged, SetParentParameters, and TempDVSCameraParameters.

Referenced by EndPlay(), and Init().

◆ SecToNanoSec()

constexpr std::int64_t ADVSCamera::SecToNanoSec ( double seconds )

inlineconstexprprivate

Convert seconds to nanoseconds.

Parameters

seconds - The time in seconds to convert.

Returns: The time in nanoseconds.

Definition at line 217 of file DVSCamera.h.

    {
        return static_cast<std::int64_t>(seconds * 1e9);
    }

Referenced by SimulateDVS().

◆ SendRawDVSImageDataToROS()

void ADVSCamera::SendRawDVSImageDataToROS ( )

private

Send raw DVS image data to ROS (Robot Operating System).

Definition at line 394 of file DVSCamera.cpp.

{
    if (EventArray.size() == 0
        || !SendDataToROS
        || !DvsTopic
        || !IsROSConnected()
        || !PointCloudMessage.IsValid())
    {
        return;
    }
 
    PointCloudMessage->width = EventArray.size();
    PointCloudMessage->row_step = PointCloudMessage->width * PointCloudMessage->point_step;
 
    const DVSEvent* PointsDataPtr = EventArray.data();
    PointCloudMessage->data_ptr = reinterpret_cast<uint8*>(const_cast<DVSEvent*>(PointsDataPtr));
 
    // Send raw DVS data (PointCloud2) to ROS Topic
    DvsTopic->Publish(PointCloudMessage);
}

References DvsTopic, EventArray, ASensor::IsROSConnected(), PointCloudMessage, and ASensor::SendDataToROS.

Referenced by AddProcessingToFrameBuffer().

◆ SimulateDVS()

std::vector< DVSEvent > ADVSCamera::SimulateDVS ( const TArray< FColor > & Buffer )

private

Definition at line 207 of file DVSCamera.cpp.

{
    std::vector<DVSEvent> Events;
 
    const int32 Width = GetCameraWidth();
    const int32 Height = GetCameraHeight();
    const int32 Size = Width * Height;
 
    if (DVSOutput.Num() != Size)
    {
        DVSOutput.Empty();
        DVSOutput.SetNumUninitialized(Size);
        DVSOutput.Init(FColor::Black, Size);
 
        PreviousImage.Empty();
        PreviousImage.SetNumZeroed(0);
    }
 
    if (Buffer.Num() != Size)
    {
        return Events;
    }
 
    if (DVSCameraParameters.UseLog)
    {
        ImageToLogGray(Buffer);
    }
    else
    {
        ImageToGray(Buffer);
    }
 
    UWorld* World = GetWorld();
 
    if (PreviousImage.Num() == 0)
    {
        // Set the first rendered image
        RefValues = LastImage;
        PreviousImage = LastImage;
 
        // Resizes array to given number of elements. New elements will be zeroed.
        LastEventTimestamp.SetNumZeroed(LastImage.Num());
 
        // Reset current time
        CurrentTime = SecToNanoSec(World->GetTimeSeconds());
 
        return Events;
    }
 
    EventArray.clear();
 
    auto DVSEventFill = DVSEvent();
    DVSEventFill.x = 100;
    DVSEventFill.y = 100;
    DVSEventFill.t = 100;
    EventArray.resize(Size, DVSEventFill);
 
    FColor desiredColor = FColor::Black;
    FMemory::Memset(DVSOutput.GetData(), desiredColor.ToPackedARGB(), DVSOutput.Num() * sizeof(FColor));
 
    static constexpr float tolerance = 1e-6;
 
    const std::uint64_t delta_t_ns = SecToNanoSec(World->GetTimeSeconds()) - CurrentTime;
 
    const float cp = DVSCameraParameters.PositiveThreshold;
    const float cm = DVSCameraParameters.NegativeThreshold;
    const float sigma_cp = DVSCameraParameters.SigmaPositiveThreshold;
    const float sigma_cm = DVSCameraParameters.SigmaNegativeThreshold;
    const float refractory_period_ns = DVSCameraParameters.RefractoryPeriodNs;
 
    // TODO: This could improved.
    for (size_t y = 0; y < Height; y++)
    {
        for (int x = 0; x < Width; ++x)
        {
            const uint32 i = (Width * y) + x;
            const float itdt = LastImage[i];
            const float it = PreviousImage[i];
            const float prev_cross = RefValues[i];
 
            if (std::fabs(it - itdt) > tolerance)
            {
                const float pol = (itdt >= it) ? +1.0 : -1.0;
                float C = (pol > 0) ? cp : cm;
                const float sigma_C = (pol > 0) ? sigma_cp : sigma_cm;
 
                if (sigma_C > 0)
                {
                    C += std::normal_distribution<float>(0, sigma_C)(Engine);
                    constexpr float minimum_contrast_threshold = 0.01;
                    C = std::max(minimum_contrast_threshold, C);
                }
                float curr_cross = prev_cross;
                bool all_crossings = false;
                do
                {
                    curr_cross += pol * C;
 
                    if ((pol > 0 && curr_cross > it && curr_cross <= itdt)
                        || (pol < 0 && curr_cross < it && curr_cross >= itdt))
                    {
                        const std::uint64_t edt = (curr_cross - it) * delta_t_ns / (itdt - it);
                        const std::int64_t t = CurrentTime + edt;
 
                        // check that pixel (x,y) is not currently in a "refractory" state
                        // i.e. |t-that last_timestamp(x,y)| >= refractory_period
                        const std::int64_t last_stamp_at_xy = SecToNanoSec(LastEventTimestamp[i]);
                        if (t >= last_stamp_at_xy)
                        {
                            const std::uint64_t dt = t - last_stamp_at_xy;
                            if (LastEventTimestamp[i] == 0 || dt >= refractory_period_ns)
                            {
                                bool val = pol > 0;
 
                                EventArray[i] = DVSEvent(x, y, t, val);
                                DVSOutput[i] = val ? FColor::Red : FColor::Black;
 
                                LastEventTimestamp[i] = NanoSecToSecTrunc(t);
                            }
                            RefValues[i] = curr_cross;
                        }
                    }
                    else
                    {
                        all_crossings = true;
                    }
                } while (!all_crossings);
            }
        }
    }
 
    // Update current time
    CurrentTime = SecToNanoSec(World->GetTimeSeconds());
 
    PreviousImage = LastImage;
 
    // Remove default values from EventArray std::vector
    EventArray.erase(std::remove(EventArray.begin(), EventArray.end(), DVSEventFill), EventArray.end());
 
    if (DVSCameraParameters.SortByIncreasingTimestamp)
    {
        // Sort the events by increasing timestamps
        std::sort(EventArray.begin(), EventArray.end(), [](const DVSEvent& it1, const DVSEvent& it2)
            {
                return it1.t < it2.t;
            });
    }
 
    return EventArray;
}

References CurrentTime, DVSCameraParameters, DVSOutput, Engine, EventArray, ACamera::GetCameraHeight(), ACamera::GetCameraWidth(), ImageToGray(), ImageToLogGray(), LastEventTimestamp, LastImage, NanoSecToSecTrunc(), FDVSCameraParameters::NegativeThreshold, FDVSCameraParameters::PositiveThreshold, PreviousImage, FDVSCameraParameters::RefractoryPeriodNs, RefValues, SecToNanoSec(), FDVSCameraParameters::SigmaNegativeThreshold, FDVSCameraParameters::SigmaPositiveThreshold, FDVSCameraParameters::SortByIncreasingTimestamp, and FDVSCameraParameters::UseLog.

Referenced by AddProcessingToFrameBuffer().

◆ UpdateDVSWindowOutput()

void ADVSCamera::UpdateDVSWindowOutput ( const TArray< FColor > & Buffer )

private

Update the DVS window output with the provided buffer and dimensions.

Parameters

Buffer - The buffer to update.

Definition at line 464 of file DVSCamera.cpp.

{
    if (!DVSUnrealWindow.IsValid() || !CaptureFrameTexture)
    {
        return;
    }
 
    // CaptureFrameTexture defined in Camera.h and it should 
    // resize automatically in Camera.cpp if parent Window is resized.
 
    // Lock texture
    uint8* TextureData = static_cast<uint8*>(CaptureFrameTexture->GetPlatformData()->Mips[0].BulkData.Lock(LOCK_READ_WRITE));
 
    // Copy buffer to texture
    FMemory::Memcpy(TextureData, Buffer.GetData(), Buffer.Num() * sizeof(FColor));
 
    // Unlock texture
    CaptureFrameTexture->GetPlatformData()->Mips[0].BulkData.Unlock();
 
    // Update texture resource
    CaptureFrameTexture->UpdateResource();
    FTextureResource* resource = CaptureFrameTexture->GetResource();
 
    // Update new texture to window and draw
    DVSUnrealWindow->UpdateTexture(resource);
    DVSUnrealWindow->DrawManually();
}

References ACamera::CaptureFrameTexture, and DVSUnrealWindow.

Referenced by AddProcessingToFrameBuffer().

Friends And Related Function Documentation

◆ USensorFactory

friend class USensorFactory

friend

Definition at line 97 of file DVSCamera.h.

Member Data Documentation

◆ ApprovedWidths

TArray<int32> ADVSCamera::ApprovedWidths

staticprivate

Definition at line 210 of file DVSCamera.h.

◆ CurrentTime

std::int64_t ADVSCamera::CurrentTime

private

Definition at line 183 of file DVSCamera.h.

Referenced by SimulateDVS().

◆ DVSCameraParameters

FDVSCameraParameters ADVSCamera::DVSCameraParameters

private

Definition at line 187 of file DVSCamera.h.

Referenced by ChangeParametersInternal(), DVSInit(), ImageToGray(), ImageToLogGray(), Init(), OnMainCameraWindowResized(), and SimulateDVS().

◆ DVSOutput

TArray<FColor> ADVSCamera::DVSOutput

private

Definition at line 198 of file DVSCamera.h.

Referenced by AddProcessingToFrameBuffer(), EndPlay(), Init(), and SimulateDVS().

◆ DvsTopic

UTopic* ADVSCamera::DvsTopic = nullptr

private

DVS Camera raw data ROS topic. Unlike other sensors, DVS camera has two topics. ROSTopic defined in sensor.h which publishes DVS camera image buffer, and this topic which publishes DVS camera raw event data as PointCloud2 message.

Definition at line 195 of file DVSCamera.h.

Referenced by CreateROSTopic(), EndPlay(), and SendRawDVSImageDataToROS().

◆ DVSUnrealWindow

TSharedPtr<FUnrealWindow> ADVSCamera::DVSUnrealWindow

private

Definition at line 204 of file DVSCamera.h.

Referenced by ChangeParametersInternal(), EndPlay(), Init(), and UpdateDVSWindowOutput().

◆ EventArray

std::vector<DVSEvent> ADVSCamera::EventArray

private

Definition at line 167 of file DVSCamera.h.

Referenced by EndPlay(), Init(), SendRawDVSImageDataToROS(), and SimulateDVS().

◆ LastEventTimestamp

TArray<double> ADVSCamera::LastEventTimestamp

private

Definition at line 172 of file DVSCamera.h.

Referenced by EndPlay(), and SimulateDVS().

◆ LastImage

TArray<float> ADVSCamera::LastImage

private

Definition at line 178 of file DVSCamera.h.

Referenced by EndPlay(), ImageToGray(), ImageToLogGray(), and SimulateDVS().

◆ OnDVSWindowClosed

FOnWindowClosed ADVSCamera::OnDVSWindowClosed

private

Definition at line 208 of file DVSCamera.h.

Referenced by EndPlay(), and Init().

◆ ParametersChanged

bool ADVSCamera::ParametersChanged = false

private

Definition at line 202 of file DVSCamera.h.

Referenced by AddProcessingToFrameBuffer(), ChangeDVSCameraParameters(), ChangeParametersInternal(), and OnMainCameraWindowResized().

◆ PointCloudMessage

TSharedPtr<ROSMessages::sensor_msgs::PointCloud2> ADVSCamera::PointCloudMessage

private

Definition at line 206 of file DVSCamera.h.

Referenced by EndPlay(), Init(), and SendRawDVSImageDataToROS().

◆ PreviousImage

TArray<float> ADVSCamera::PreviousImage

private

Definition at line 175 of file DVSCamera.h.

Referenced by EndPlay(), and SimulateDVS().

◆ RefValues

TArray<float> ADVSCamera::RefValues

private

Definition at line 181 of file DVSCamera.h.

Referenced by EndPlay(), and SimulateDVS().

◆ SetParentParameters

bool ADVSCamera::SetParentParameters = false

private

Definition at line 200 of file DVSCamera.h.

Referenced by ChangeDVSCameraParameters(), ChangeParametersInternal(), and OnMainCameraWindowResized().

◆ TempDVSCameraParameters

FDVSCameraParameters ADVSCamera::TempDVSCameraParameters

private

Definition at line 185 of file DVSCamera.h.

Referenced by ChangeDVSCameraParameters(), ChangeParametersInternal(), and OnMainCameraWindowResized().

The documentation for this class was generated from the following files:

Unreal/Agrarsense/Source/Agrarsense/Sensor/Camera/DVSCamera.h
Unreal/Agrarsense/Source/Agrarsense/Sensor/Camera/DVSCamera.cpp

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Static Private Attributes

Friends

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ADVSCamera()

Member Function Documentation

◆ AddProcessingToFrameBuffer()

◆ ChangeDVSCameraParameters()

◆ ChangeParametersInternal()

◆ CreateROSTopic()

◆ DVSInit()

◆ DVSWindowClosedCallback()

◆ EndPlay()

◆ FColorToGrayScaleFloat()

◆ GetDVSCameraParameters()

◆ GetDVSTopic()

◆ GetDVSTopicName()

◆ GetParametersAsString()

◆ GetSensorType()

◆ ImageToGray()

◆ ImageToLogGray()

◆ Init()

◆ NanoSecToSecTrunc()

◆ OnMainCameraWindowResized()

◆ SecToNanoSec()

◆ SendRawDVSImageDataToROS()

◆ SimulateDVS()

◆ UpdateDVSWindowOutput()

Friends And Related Function Documentation

◆ USensorFactory

Member Data Documentation

◆ ApprovedWidths

◆ CurrentTime

◆ DVSCameraParameters

◆ DVSOutput

◆ DvsTopic

◆ DVSUnrealWindow

◆ EventArray

◆ LastEventTimestamp

◆ LastImage

◆ OnDVSWindowClosed

◆ ParametersChanged

◆ PointCloudMessage

◆ PreviousImage

◆ RefValues

◆ SetParentParameters

◆ TempDVSCameraParameters