Agrarsense
Camera.cpp
Go to the documentation of this file.
1// Copyright (c) 2023 FrostBit Software Lab at the Lapland University of Applied Sciences
2//
3// This work is licensed under the terms of the MIT license.
4// For a copy, see <https://opensource.org/licenses/MIT>.
5
6#include "Camera.h"
7
14
15#include "NiagaraComponent.h"
16#include "Components/PrimitiveComponent.h"
17#include "Materials/Material.h"
18#include "ImageUtils.h"
19#include "Async/Async.h"
20#include "HighResScreenshot.h"
21#include "ImageWriteTask.h"
22#include "ImageWriteQueue.h"
23#include "Misc/DateTime.h"
24#include "Math/Color.h"
25#include "Blueprint/UserWidget.h"
26#include "GameFramework/PlayerController.h"
27#include "GeoReferencingSystem.h"
28#include "TimerManager.h"
29
30#include "GameFramework/SpringArmComponent.h"
32
33ACamera::ACamera(const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer)
34{
35 PrimaryActorTick.bCanEverTick = true;
36 PrimaryActorTick.TickGroup = TG_PostUpdateWork;
37
38 CaptureRenderTarget = CreateDefaultSubobject<UTextureRenderTarget2D>(FName(*FString::Printf(TEXT("CaptureRenderTarget"))));
39 CaptureRenderTarget->CompressionSettings = TextureCompressionSettings::TC_Default;
40 CaptureRenderTarget->SRGB = false;
41 CaptureRenderTarget->bAutoGenerateMips = false;
42 CaptureRenderTarget->bGPUSharedFlag = true;
43 CaptureRenderTarget->AddressX = TextureAddress::TA_Clamp;
44 CaptureRenderTarget->AddressY = TextureAddress::TA_Clamp;
45
46 CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(FName(*FString::Printf(TEXT("SceneCaptureComponent2D"))));
47 CaptureComponent2D->SetupAttachment(RootComponent);
48 CaptureComponent2D->PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_RenderScenePrimitives;
49 CaptureComponent2D->bCaptureOnMovement = false;
50 CaptureComponent2D->bCaptureEveryFrame = false;
51 CaptureComponent2D->bAlwaysPersistRenderingState = true;
52}
53
55{
56 SetupCamera(newParameters);
57}
58
59void ACamera::AddPostProcessingMaterial(const FString& Path, float Weight)
60{
61 UMaterial* PostProcessingMat = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(), nullptr, *Path));
62 if (PostProcessingMat && CaptureComponent2D)
63 {
64 FPostProcessSettings& PostProcessSettings = CaptureComponent2D->PostProcessSettings;
65 PostProcessSettings.AddBlendable(PostProcessingMat, Weight);
66 }
67 else
68 {
69 FString Sensor = GetSensorName();
70 FString Msg = "Failed to add Post processing material to " + Sensor;
72 }
73}
74
76{
78 {
79 FPostProcessSettings& PostProcessSettings = CaptureComponent2D->PostProcessSettings;
80 PostProcessSettings.RemoveBlendable(Material);
81 }
82}
83
84void ACamera::Init(FCameraBaseParameters parameters, bool SimulateSensor)
85{
87 SetSimulateSensor(SimulateSensor);
88 SetupCamera(parameters);
90}
91
93{
94 if (IsValid(LogFile))
95 {
96 // File has already been created, return
97 return;
98 }
99
100 FLogFileSettings Settings;
103 Settings.QueueLength = MAX_int32; // Only write the log after destroying the sensor
104 Settings.KeepFileOpen = false;
105 Settings.Timestamp = false;
106 Settings.OverrideFilePath = true;
107 Settings.FilePath = FileSavePath;
108
109 LogFile = NewObject<ULogFile>(ULogFile::StaticClass());
110 if (IsValid(LogFile))
111 {
112 FString FileName = "camera_metadata";
113 LogFile->Create(FileName, Settings);
114
115 // Write camera metadata first line to explain the values.
116 GeoReferencingSystem = AGeoReferencingSystem::GetGeoReferencingSystem(GetWorld());
118 {
119 WriteToLogFile("image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation, GPS latitude, GPS longitude, GPS altitude");
120 }
121 else
122 {
123 WriteToLogFile("image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation");
124 }
125 }
126}
127
129{
131 {
132#if WITH_EDITOR
133 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: CaptureComponent2D is nullptr!"));
134#endif
135 return;
136 }
137
138 CameraParameters = parameters;
139
141
143 {
145 }
146
147 const bool UsePostProcessingEffects = CameraParameters.PostProcessingEffects;
148 auto& PostProcessSettings = CaptureComponent2D->PostProcessSettings;
149
150 if (parameters.UsePhysicLensDistortionEffect)
151 {
152 if (!PhysicLensDistortion.IsValid())
153 {
154 const FString Path = "/Game/Agrarsense/Materials/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion";
155 PhysicLensDistortion = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(), nullptr, *Path));
156
157 if (PhysicLensDistortion.IsValid())
158 {
159 PostProcessSettings.AddBlendable(PhysicLensDistortion.Get(), 1.0f);
160#if WITH_EDITOR
161 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Added physics lens distortion effect."));
162#endif
163 }
164 }
165 }
166 else if (PhysicLensDistortion.IsValid())
167 {
168 PostProcessSettings.RemoveBlendable(PhysicLensDistortion.Get());
169 PhysicLensDistortion.Reset();
170#if WITH_EDITOR
171 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Removed physics lens distortion effect."));
172#endif
173 }
174
175 if (parameters.UseIceLensEffect)
176 {
177 if (!IceMaterialInstance.IsValid())
178 {
179 const FString Path = "/Game/Agrarsense/Materials/PostProcessingMaterials/CameraPostProcessEffects/m_ice_lens_effect";
180 UMaterial* LoadedIceMaterial = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(), nullptr, *Path));
181
182 // Create UMaterialInstanceDynamic from LoadedIceMaterial
183 IceMaterialInstance = UMaterialInstanceDynamic::Create(LoadedIceMaterial, nullptr);
184 if (IceMaterialInstance.IsValid())
185 {
186 // Add material to CaptureComponent2D PostProcessSettings
187 PostProcessSettings.AddBlendable(IceMaterialInstance.Get(), 1.0f);
188#if WITH_EDITOR
189 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Added ice lens effect."));
190#endif
191 }
192 }
193
194 UMaterialInstanceDynamic* IceMatInstance = IceMaterialInstance.Get();
195 if (IceMatInstance)
196 {
197 // Set IceMaterialInstance scalar parameters values
198 IceMatInstance->SetScalarParameterValue(FName("Strength"), parameters.IceLensEffectStrength);
199 IceMatInstance->SetScalarParameterValue(FName("Angle"), parameters.IceLensEffectAngle);
200#if WITH_EDITOR
201 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Changed ice lens effect parameters."));
202#endif
203 }
204 }
205 else if (IceMaterialInstance.IsValid())
206 {
207 UMaterialInstanceDynamic* IceMatInstance = IceMaterialInstance.Get();
208 if (IceMatInstance)
209 {
210 // Remove material from CaptureComponent2D PostProcessSettings
211 PostProcessSettings.RemoveBlendable(IceMatInstance);
212 IceMaterialInstance.Reset();
213
214#if WITH_EDITOR
215 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Removed ice lens effect."));
216#endif
217 }
218 }
219
222 const float TargetGamma = CameraParameters.TargetGamma;
223 const bool Enable16BitFormat = CameraParameters.Enable16BitFormat;
224
225 CaptureRenderTarget->InitCustomFormat(ImageWidth, ImageHeight, Enable16BitFormat ? PF_FloatRGB : PF_B8G8R8A8, !UsePostProcessingEffects);
226
227 CaptureRenderTarget->TargetGamma = TargetGamma;
228
229 // Set PostProcessSettings override (on or off)
230 PostProcessSettings.bOverride_AutoExposureMethod = UsePostProcessingEffects;
231 PostProcessSettings.bOverride_AutoExposureBias = UsePostProcessingEffects;
232 PostProcessSettings.bOverride_AutoExposureMinBrightness = UsePostProcessingEffects;
233 PostProcessSettings.bOverride_AutoExposureMaxBrightness = UsePostProcessingEffects;
234 PostProcessSettings.bOverride_AutoExposureSpeedUp = UsePostProcessingEffects;
235 PostProcessSettings.bOverride_AutoExposureSpeedDown = UsePostProcessingEffects;
236 PostProcessSettings.bOverride_HistogramLogMin = UsePostProcessingEffects;
237 PostProcessSettings.bOverride_HistogramLogMax = UsePostProcessingEffects;
238 PostProcessSettings.bOverride_CameraShutterSpeed = UsePostProcessingEffects;
239 PostProcessSettings.bOverride_CameraISO = UsePostProcessingEffects;
240 PostProcessSettings.bOverride_DepthOfFieldFstop = UsePostProcessingEffects;
241 PostProcessSettings.bOverride_DepthOfFieldMinFstop = UsePostProcessingEffects;
242 PostProcessSettings.bOverride_DepthOfFieldBladeCount = UsePostProcessingEffects;
243 PostProcessSettings.bOverride_FilmSlope = UsePostProcessingEffects;
244 PostProcessSettings.bOverride_FilmToe = UsePostProcessingEffects;
245 PostProcessSettings.bOverride_FilmShoulder = UsePostProcessingEffects;
246 PostProcessSettings.bOverride_FilmWhiteClip = UsePostProcessingEffects;
247 PostProcessSettings.bOverride_FilmBlackClip = UsePostProcessingEffects;
248 PostProcessSettings.bOverride_MotionBlurAmount = UsePostProcessingEffects;
249 PostProcessSettings.bOverride_MotionBlurMax = UsePostProcessingEffects;
250 PostProcessSettings.bOverride_MotionBlurPerObjectSize = UsePostProcessingEffects;
251 PostProcessSettings.bOverride_WhiteTemp = UsePostProcessingEffects;
252 PostProcessSettings.bOverride_WhiteTint = UsePostProcessingEffects;
253 PostProcessSettings.bOverride_ColorContrast = UsePostProcessingEffects;
254 PostProcessSettings.bOverride_SceneFringeIntensity = UsePostProcessingEffects;
255 PostProcessSettings.bOverride_ChromaticAberrationStartOffset = UsePostProcessingEffects;
256 PostProcessSettings.bOverride_AmbientOcclusionIntensity = UsePostProcessingEffects;
257 PostProcessSettings.bOverride_AmbientOcclusionRadius = UsePostProcessingEffects;
258 PostProcessSettings.bOverride_AmbientOcclusionStaticFraction = UsePostProcessingEffects;
259 PostProcessSettings.bOverride_AmbientOcclusionFadeDistance = UsePostProcessingEffects;
260 PostProcessSettings.bOverride_AmbientOcclusionPower = UsePostProcessingEffects;
261 PostProcessSettings.bOverride_AmbientOcclusionBias = UsePostProcessingEffects;
262 PostProcessSettings.bOverride_AmbientOcclusionQuality = UsePostProcessingEffects;
263 PostProcessSettings.bOverride_BloomMethod = UsePostProcessingEffects;
264 PostProcessSettings.bOverride_BloomIntensity = UsePostProcessingEffects;
265 PostProcessSettings.bOverride_BloomThreshold = UsePostProcessingEffects;
266 PostProcessSettings.bOverride_LensFlareIntensity = UsePostProcessingEffects;
267 PostProcessSettings.bOverride_DepthOfFieldFocalDistance = UsePostProcessingEffects;
268 PostProcessSettings.bOverride_DepthOfFieldDepthBlurAmount = UsePostProcessingEffects;
269 PostProcessSettings.bOverride_DepthOfFieldDepthBlurRadius = UsePostProcessingEffects;
270
271 // Set PostProcessSettings values
272 PostProcessSettings.CameraShutterSpeed = CameraParameters.ShutterSpeed;
273 PostProcessSettings.CameraISO = CameraParameters.ISO;
274 PostProcessSettings.DepthOfFieldFstop = CameraParameters.Aperture;
275 PostProcessSettings.DepthOfFieldFocalDistance = CameraParameters.FocalDistance;
276 PostProcessSettings.DepthOfFieldDepthBlurAmount = CameraParameters.DepthBlurAmount;
277 PostProcessSettings.DepthOfFieldDepthBlurRadius = CameraParameters.DepthBlurRadius;
278 PostProcessSettings.DepthOfFieldMinFstop = CameraParameters.DofMinFStop;
279 PostProcessSettings.DepthOfFieldBladeCount = CameraParameters.DofBladeCount;
280 PostProcessSettings.FilmSlope = CameraParameters.FilmSlope;
281 PostProcessSettings.FilmToe = CameraParameters.FilmToe;
282 PostProcessSettings.FilmShoulder = CameraParameters.FilmShoulder;
283 PostProcessSettings.FilmBlackClip = CameraParameters.FilmBlackClip;
284 PostProcessSettings.FilmWhiteClip = CameraParameters.FilmWhiteClip;
285 PostProcessSettings.AutoExposureMinBrightness = CameraParameters.ExposureMinBrightness;
286 PostProcessSettings.AutoExposureMaxBrightness = CameraParameters.ExposureMaxBrightness;
287 PostProcessSettings.AutoExposureSpeedUp = CameraParameters.ExposureSpeedUp;
288 PostProcessSettings.AutoExposureSpeedDown = CameraParameters.ExposureSpeedDown;
289 PostProcessSettings.MotionBlurAmount = CameraParameters.MotionBlurIntensity;
290 PostProcessSettings.MotionBlurMax = CameraParameters.MotionBlurMax;
291 PostProcessSettings.MotionBlurPerObjectSize = CameraParameters.MotionBlurMinObjSize;
292 PostProcessSettings.LensFlareIntensity = CameraParameters.LensFlareIntensity;
293 PostProcessSettings.BloomIntensity = CameraParameters.BloomIntensity;
294 PostProcessSettings.WhiteTemp = CameraParameters.WhiteTemp;
295 PostProcessSettings.WhiteTint = CameraParameters.WhiteTint;
296 PostProcessSettings.SceneFringeIntensity = CameraParameters.ChromAberrIntensity;
297 PostProcessSettings.ChromaticAberrationStartOffset = CameraParameters.ChromAberrOffset;
298
301
303 {
304 // Enable HDR on CaptureSource, only allowed for RGB Camera for now.
305 CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_FinalColorHDR;
306 }
307 else
308 {
309 CaptureComponent2D->CaptureSource = ESceneCaptureSource::SCS_FinalColorLDR;
310 }
311
312 CaptureComponent2D->bUseRayTracingIfEnabled = true;
313 CaptureComponent2D->UpdateContent();
314 CaptureComponent2D->Activate();
315
317
318 FString Encoding = "bgr8";
319
320 // Create ROS Image message
321 ImageMsg.Reset();
322 ImageMsg = MakeShared<ROSMessages::sensor_msgs::Image>();
323 ImageMsg->header.frame_id = "world";
324 ImageMsg->height = ImageWidth;
325 ImageMsg->width = ImageHeight;
326 ImageMsg->encoding = Encoding;
327 ImageMsg->is_bigendian = 0;
328 ImageMsg->step = ImageWidth * 4;
329
331
332 if (!UnrealWindow.IsValid())
333 {
334 // Create and setup UnrealWindow
335 FString windowName = GetSensorIdentifier() + " Window";
336 UnrealWindow = MakeShareable(new FUnrealWindow(ImageWidth, ImageHeight, windowName));
337
338 // Set on Window closed event
340 UnrealWindow->GetSWindow()->SetOnWindowClosed(OnWindowClosedDelegate);
341
342 // try to create Camera parameter Widget from path
343 FSoftClassPath WidgetClassPath(TEXT("/Game/Agrarsense/Widgets/Camera/WBP_CameraControlsMenu.WBP_CameraControlsMenu_C"));
344 UClass* WidgetClass = WidgetClassPath.TryLoadClass<UUserWidget>();
345
346 APlayerController* PlayerController = UGameplayStatics::GetPlayerController(GetWorld(), 0);
347
348 if (WidgetClass && PlayerController)
349 {
350 UCameraWidgetBase* WidgetInstance = CreateWidget<UCameraWidgetBase>(PlayerController, WidgetClass);
351 if (WidgetInstance)
352 {
353 bool ShowGuide = false;
354 WidgetInstance->Setup(this, ShowGuide);
355 this->AddWidgetToWindow(WidgetInstance);
356 }
357 }
358 }
359
360 bool Resized = false;
361 if (UnrealWindow.IsValid())
362 {
363 UnrealWindow->SetupComponent(CaptureRenderTarget);
364 if (UnrealWindow->GetWindowWidth() != ImageWidth || UnrealWindow->GetWindowHeight() != ImageHeight)
365 {
366 UnrealWindow->ResizeWindow(ImageWidth, ImageHeight);
367 Resized = true;
368 }
369 }
370
372
373 if (Resized)
374 {
376 }
377}
378
380{
381 Super::BeginPlay();
382
383 // Register to PreActorTick where USceneCaptureComponent2D will get rendered.
384 OnPreTickDelegate = FWorldDelegates::OnWorldPreActorTick.AddUObject(this, &ACamera::PreActorTick);
385
386 // Check if we should use ATickManager which ticks in parallel, or use Post actor tick
387 // In there we capture the screen pixels with FrameGrabber API,
388 // might do some extra processing, save image to disk and send data to ROS.
390 {
391 SetActorTickEnabled(false);
392 TickEntry = ATickManager::AddTick(this, BindTick(this, &ACamera::EndOfFrameParellel), ETickType::LateTickParallel);
393 }
394 else
395 {
396 SetActorTickEnabled(true);
397 OnPostTickDelegate = FWorldDelegates::OnWorldPostActorTick.AddUObject(this, &ACamera::EndOfFrame);
398 }
399
400 // Get all existing components set to be hidden for all Camera sensors (like Lidar point cloud visualization).
401 auto Components = ASensor::GetComponentsToHide();
402 for (int32 i = 0; i < Components.Num(); i++)
403 {
404 UPrimitiveComponent* Primitive = Components[i].Get();
405 if (Primitive)
406 {
407 HidePrimitiveComponent(Primitive);
408 }
409 }
410
411 OnPrimitiveAdded.AddUniqueDynamic(this, &ACamera::HidePrimitiveComponent);
412
413 GetWorld()->GetTimerManager().SetTimerForNextTick([this]()
414 {
415 if (IsValid(this))
416 {
417 DronePtr = Cast<APIDDrone>(GetAttachParentActor());
418 if (!DronePtr)
419 {
420 AllowGimbal = false;
421 }
422 }
423 });
424}
425
426void ACamera::EndPlay(const EEndPlayReason::Type EndPlayReason)
427{
428 Super::EndPlay(EndPlayReason);
429
430 FWorldDelegates::OnWorldPreActorTick.Remove(OnPreTickDelegate);
431
433 {
435 }
436 else
437 {
438 FWorldDelegates::OnWorldPostActorTick.Remove(OnPostTickDelegate);
439 }
440
441 if (UnrealWindow.IsValid())
442 {
443 OnWindowClosedDelegate.Unbind();
444 UnrealWindow->DestroyWindow();
445 UnrealWindow.Reset();
446 }
447
449
451 {
452 CaptureRenderTarget->ConditionalBeginDestroy();
453 CaptureRenderTarget = nullptr;
454 }
455
457 {
458 CaptureComponent2D->UnregisterComponent();
459 CaptureComponent2D->ConditionalBeginDestroy();
460 CaptureComponent2D = nullptr;
461 }
462
463 ImageMsg.Reset();
464 IceMaterialInstance.Reset();
465
466 if (BGR8Buffer)
467 {
468 delete[] BGR8Buffer;
469 BGR8Buffer = nullptr;
470 }
472}
473
474void ACamera::PreActorTick(UWorld* World, ELevelTick TickType, float DeltaSeconds)
475{
476 TRACE_CPUPROFILER_EVENT_SCOPE(ACamera::PreActorTick);
477
478 // If this Camera sensor is attached to Drone, keep the camera steady kind of like a Gimbal.
480 {
481 if (SensorSetRotation == FRotator::ZeroRotator)
482 {
483 // Save garage sensor values
484 SensorSetRotation = GetTransform().Rotator();
485 }
486 FRotator gimbalRot = FRotator(SensorSetRotation.Pitch, GetActorRotation().Yaw, SensorSetRotation.Roll);
487 SetActorRotation(gimbalRot);
488 }
489
490 if (ShouldSimulate(DeltaSeconds) && CaptureComponent2D)
491 {
492 // Capture scene now.
493 // We could use CaptureSceneDeferred but it would not work if World (Spectator) Rendering is disabled.
494 CaptureComponent2D->CaptureScene();
495
496 // Inform FrameGrabber to capture this frame
497 if (FrameGrabber.IsValid())
498 {
499 FrameGrabber->CaptureThisFrame(FFramePayloadPtr());
500 }
501 }
502}
503
504void ACamera::EndOfFrame(UWorld* World, ELevelTick TickType, float DeltaSeconds)
505{
507}
508
509void ACamera::EndOfFrameParellel(float DeltaTime)
510{
512}
513
514bool ACamera::ShouldSimulate(const float DeltaSeconds)
515{
516 if (!CanSimulateSensor())
517 {
518 ShouldSimulateCamera = false;
520 }
521
523 {
526 }
527
528 FrameRateTimer += DeltaSeconds;
530 {
531 FrameRateTimer = 0.0f;
534 }
535
536 ShouldSimulateCamera = false;
537
539}
540
542{
543 TRACE_CPUPROFILER_EVENT_SCOPE(ACamera::FrameGrabberCapture);
544
545 if (!ShouldSimulateCamera || !FrameGrabber.IsValid())
546 {
547 return;
548 }
549
551 {
552 return;
553 }
554
555 TArray<FCapturedFrameData> Frames = FrameGrabber->GetCapturedFrames();
556 if (Frames.Num())
557 {
558 FCapturedFrameData& LastFrame = Frames.Last();
559 TArray<FColor>& ImageBuffer = LastFrame.ColorBuffer;
560
561 AddProcessingToFrameBuffer(ImageBuffer);
562
564 {
566 AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [this, ImageBuffer]()
567 {
569 });
570 }
571
573 }
574}
575
576void ACamera::AddProcessingToFrameBuffer(TArray<FColor>& buffer)
577{
578 // Only DVSCamera adds processing to the frame buffer
579 return;
580}
581
582void ACamera::HidePrimitiveComponent(UPrimitiveComponent* PrimitiveComponent)
583{
584 if (PrimitiveComponent && CaptureComponent2D)
585 {
586 CaptureComponent2D->HideComponent(PrimitiveComponent);
587 }
588}
589
591{
592 if (!UnrealWindow.IsValid())
593 {
594#if WITH_EDITOR
595 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: UnrealWindow is nullptr!"));
596#endif
597 return;
598 }
599
600 // Destroy old FrameGrabber
602
603 TSharedPtr<FSceneViewport> SceneViewport = UnrealWindow->GetSceneViewport();
604 if (SceneViewport.IsValid() && CaptureRenderTarget)
605 {
606 EPixelFormat PixelFormat = GetPixelFormatFromRenderTargetFormat(TextureFormat);
607 FIntPoint Size = SceneViewport->GetSize();
608
610 {
611 // Currently only DVS camera needs this texture to render DVS output into separate FUnrealWindow.
612 CaptureFrameTexture = UTexture2D::CreateTransient(Size.X, Size.Y, PixelFormat);
613 CaptureFrameTexture->UpdateResource();
614 }
615
616 FrameGrabber = MakeShareable(new FFrameGrabber(SceneViewport.ToSharedRef(), Size, PixelFormat));
617 FrameGrabber->StartCapturingFrames();
618 }
619}
620
622{
623 if (FrameGrabber.IsValid())
624 {
625 FrameGrabber->StopCapturingFrames();
626 FrameGrabber->Shutdown();
627 FrameGrabber.Reset();
628 }
629
631 {
632 CaptureFrameTexture->RemoveFromRoot();
633 CaptureFrameTexture->ConditionalBeginDestroy();
634 CaptureFrameTexture = nullptr;
635 }
636}
637
638void ACamera::SendImageDataToROS(const TArray<FColor>& FrameBuffer, int32 Width, int32 Height)
639{
640 TRACE_CPUPROFILER_EVENT_SCOPE(ACamera::SendImageDataToROS);
641
642 UTopic* Topic = GetROSTopic();
643 if (!SendDataToROS
644 || !Topic
645 || !IsROSConnected()
646 || FrameBuffer.IsEmpty()
647 || !ImageMsg.IsValid())
648 {
649 return;
650 }
651
652 ImageMsg->height = Height;
653 ImageMsg->width = Width;
654
655 // Cast TArray<FColor> into uint8[] data with all channels (RBGA)
656 // Currently not in use but left here incase this is ever needed again.
657 //ImageMsg->encoding = "bgra8";
658 //ImageMsg->step = Width * 4;
659 //ImageMsg->data = const_cast<uint8*>(reinterpret_cast<const uint8*>(FrameBuffer.GetData()));
660
661 // Convert TArray<FColor> to uint8[] data without alpha channel
662 // By removing alpha channel, we reduce ~25% of ROS bandwidth,
663 // but costs a bit more in processing time here
664 const int32 BufferSize = Width * Height * 3;
665 if (!BGR8Buffer || CurrentBufferSize != BufferSize)
666 {
667 // Resize buffer if needed
668 delete[] BGR8Buffer;
669 BGR8Buffer = new uint8[BufferSize];
670 CurrentBufferSize = BufferSize;
671 }
672
673 uint8* Dest = BGR8Buffer;
674
675 const FColor* Source = FrameBuffer.GetData();
676 const int32 PixelCount = FrameBuffer.Num();
677
678 for (int32 i = 0; i < PixelCount; ++i)
679 {
680 const FColor& Pixel = Source[i];
681 *Dest++ = Pixel.B;
682 *Dest++ = Pixel.G;
683 *Dest++ = Pixel.R;
684 }
685
686 ImageMsg->step = Width * 3;
687 ImageMsg->data = BGR8Buffer;
688
689 // Send Image to ROS
690 Topic->Publish(ImageMsg);
691}
692
693void ACamera::SaveImageToDisk(const TArray<FColor> FrameBuffer, int32 Width, int32 Height)
694{
695 TRACE_CPUPROFILER_EVENT_SCOPE(ACamera::SaveImageToDisk);
696
697 if (FrameBuffer.IsEmpty() || Width == 0 || Height == 0)
698 {
699 return;
700 }
701
702 int32 Size = Width * Height;
703 if (FrameBuffer.Num() != Size)
704 {
705 FString Msg = "Camera sensor: Unable to save Image to disk. FrameBuffer size and resolution don't match!";
707 return;
708 }
709
710 FString ImageName = FString::Printf(TEXT("image_%d.png"), ImageNumber);
711 FString fullFileName = FString::Printf(TEXT("%s%s"), *FileSavePath, *ImageName);
712 ++ImageNumber;
713
714 SaveCameraMetaDataToDisk(ImageName);
715
716 FIntPoint DestSize(Width, Height);
717 TImagePixelData<FColor> PixelData(DestSize);
718 PixelData.Pixels = FrameBuffer;
719
720 // Create ImageTask
721 TUniquePtr<FImageWriteTask> ImageTask = MakeUnique<FImageWriteTask>();
722 ImageTask->PixelData = MakeUnique<TImagePixelData<FColor>>(PixelData);
723 ImageTask->Filename = fullFileName;
724 ImageTask->Format = EImageFormat::PNG;
725 ImageTask->CompressionQuality = (int32)EImageCompressionQuality::Default;
726 ImageTask->bOverwriteFile = true;
727 ImageTask->PixelPreProcessors.Add(TAsyncAlphaWrite<FColor>(255));
728
729 // Get Screenshot config and enqueue image save task
730 FHighResScreenshotConfig& HighResScreenshotConfig = GetHighResScreenshotConfig();
731 TFuture<bool> CompletionFuture = HighResScreenshotConfig.ImageWriteQueue->Enqueue(MoveTemp(ImageTask));
732}
733
734void ACamera::SaveCameraMetaDataToDisk(const FString& ImageName)
735{
736 if (!IsValid(LogFile))
737 {
738 // If the log file located in base class is not valid, return here.
739 return;
740 }
741
742 const FVector ActorPosition = GetActorLocation();
743 const FRotator ActorRotation = GetActorRotation();
744
745 FString MetaData;
746
748 {
749 FGeographicCoordinates GeoCoordinates = UCoordinateConversionUtilities::UnrealToGeographicCoordinates(GeoReferencingSystem, ActorPosition);
750 MetaData = FString::Printf(TEXT("%s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.6f, %.6f, %.2f"), *ImageName, ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
751 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll,
752 GeoCoordinates.Latitude, GeoCoordinates.Longitude, GeoCoordinates.Altitude);
753 }
754 else
755 {
756 MetaData = FString::Printf(TEXT("%s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f"), *ImageName, ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
757 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll);
758 }
759
760 // Write to the log file (this is written after sensor is destroyed)
761 WriteToLogFile(MetaData);
762}
763
764void ACamera::OnWindowClosed(const TSharedRef<SWindow>& Window)
765{
766#if WITH_EDITOR
767 UE_LOG(LogTemp, Warning, TEXT("Camera.cpp: Camera window closed. Destroying Camera sensor.."));
768#endif
769
770 // Broadcast event that this Camera sensor is about be destroyed when user has clicked Window Close button.
771 OnCameraWindowClosed.Broadcast(this);
772
773 // Destroy this camera sensor
774 this->Destroy();
775}
776
777void ACamera::AddWidgetToWindow(UWidget* WidgetToAdd)
778{
779 if (UnrealWindow.IsValid() && WidgetToAdd)
780 {
781 UnrealWindow->AddUWidgetToWindow(WidgetToAdd);
782 }
783}
784
785void ACamera::RemoveWidgetFromWindow(UWidget* WidgetToRemove)
786{
787 if (UnrealWindow.IsValid() && WidgetToRemove)
788 {
789 UnrealWindow->RemoveUWidgetFromWindow(WidgetToRemove);
790 }
791}
792
793void ACamera::SetShadowRendering(bool RenderShadows)
794{
796 {
797 auto& CameraShowFlags = CaptureComponent2D->ShowFlags;
798 CameraShowFlags.SetDynamicShadows(RenderShadows);
799 CameraShowFlags.SetContactShadows(RenderShadows);
800 CameraShowFlags.SetCapsuleShadows(RenderShadows);
801 }
802}
803
804void ACamera::SetTemporalAA(bool SetTemporal)
805{
807 {
808 auto& CameraShowFlags = CaptureComponent2D->ShowFlags;
809 CameraShowFlags.SetTemporalAA(SetTemporal);
810 }
811}
812
813void ACamera::ResizeCamera(int Width, int Height)
814{
815 if (Width == 0 || Height == 0)
816 {
817 return;
818 }
819
820 CameraParameters.Width = Width;
821 CameraParameters.Height = Height;
823}
static auto BindTick(ObjectType *Object, FunctionType Function)
Definition: TickManager.h:182
bool UseParallelLateTick
Definition: Camera.h:227
void AddPostProcessingMaterial(const FString &Path, float Weight=1.0f)
Definition: Camera.cpp:59
FTickEntry TickEntry
Definition: Camera.h:288
TSharedPtr< FFrameGrabber > FrameGrabber
Definition: Camera.h:325
void ChangeCameraParameters(FCameraBaseParameters newParameters)
Definition: Camera.cpp:54
virtual ESensorTypes GetSensorType() const override
Definition: Camera.h:74
FCameraBaseParameters CameraParameters
Definition: Camera.h:341
FOnWindowClosed OnWindowClosedDelegate
Definition: Camera.h:354
virtual void Init(FCameraBaseParameters parameters, bool SimulateSensor=true)
Definition: Camera.cpp:84
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override
Definition: Camera.cpp:426
APIDDrone * DronePtr
Definition: Camera.h:295
int ImageWidth
Definition: Camera.h:358
USceneCaptureComponent2D * CaptureComponent2D
Definition: Camera.h:277
void RemovePostProcessingMaterial(UMaterial *Material)
Definition: Camera.cpp:75
void SaveCameraMetaDataToDisk(const FString &ImageName)
Definition: Camera.cpp:734
AGeoReferencingSystem * GeoReferencingSystem
Definition: Camera.h:350
float FrameRateTimer
Definition: Camera.h:366
void ResizeCamera(int Width=1280, int Height=720)
Definition: Camera.cpp:813
void SaveImageToDisk(const TArray< FColor > FrameBuffer, int32 Width, int32 Height)
Definition: Camera.cpp:693
FDelegateHandle OnPostTickDelegate
Definition: Camera.h:232
bool ShouldSimulateCamera
Definition: Camera.h:363
void SetShadowRendering(bool RenderShadows)
Definition: Camera.cpp:793
void OnWindowClosed(const TSharedRef< SWindow > &Window)
Definition: Camera.cpp:764
void StartFrameGrabber()
Definition: Camera.cpp:590
void FrameGrabberCapture()
Definition: Camera.cpp:541
virtual void SendImageDataToROS(const TArray< FColor > &FrameBuffer, int32 Width, int32 Height)
Definition: Camera.cpp:638
FDelegateHandle OnPreTickDelegate
Definition: Camera.h:229
int ImageHeight
Definition: Camera.h:359
UTextureRenderTarget2D * CaptureRenderTarget
Definition: Camera.h:274
void AddWidgetToWindow(UWidget *WidgetToAdd)
Definition: Camera.cpp:777
TSharedPtr< ROSMessages::sensor_msgs::Image > ImageMsg
Definition: Camera.h:356
int32 CurrentBufferSize
Definition: Camera.h:372
FRotator SensorSetRotation
Definition: Camera.h:296
bool SaveCurrentFrameToDiskRequested
Definition: Camera.h:361
void SetTemporalAA(bool SetTemporal)
Definition: Camera.cpp:804
void CreateLogFile() override
Definition: Camera.cpp:92
UTexture2D * CaptureFrameTexture
Definition: Camera.h:284
void SetupCamera(FCameraBaseParameters parameters)
Definition: Camera.cpp:128
TWeakObjectPtr< UMaterial > PhysicLensDistortion
Definition: Camera.h:292
void RemoveWidgetFromWindow(UWidget *WidgetToRemove)
Definition: Camera.cpp:785
FCameraDelegate_OnWindowResized OnCameraWindowResized
Definition: Camera.h:148
void HidePrimitiveComponent(UPrimitiveComponent *PrimitiveComponent)
Definition: Camera.cpp:582
TEnumAsByte< ETextureRenderTargetFormat > TextureFormat
Definition: Camera.h:134
void ReleaseFrameGrabber()
Definition: Camera.cpp:621
virtual void EndOfFrameParellel(float DeltaTime)
Definition: Camera.cpp:509
virtual void AddProcessingToFrameBuffer(TArray< FColor > &buffer)
Definition: Camera.cpp:576
uint8 * BGR8Buffer
Definition: Camera.h:370
TSharedPtr< FUnrealWindow > UnrealWindow
Definition: Camera.h:286
virtual void EndOfFrame(UWorld *World, ELevelTick TickType, float DeltaSeconds)
Definition: Camera.cpp:504
float CameraFrameRate
Definition: Camera.h:365
ACamera(const FObjectInitializer &ObjectInitializer)
Definition: Camera.cpp:33
virtual void PreActorTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
Definition: Camera.cpp:474
FCameraDelegate_OnWindowClosed OnCameraWindowClosed
Definition: Camera.h:141
virtual void BeginPlay() override
Definition: Camera.cpp:379
TWeakObjectPtr< UMaterialInstanceDynamic > IceMaterialInstance
Definition: Camera.h:290
bool ShouldSimulate(const float DeltaSeconds)
Definition: Camera.cpp:514
int ImageNumber
Definition: Camera.h:368
bool AllowGimbal
Definition: Camera.h:297
bool CanSimulateSensor() const
Definition: Sensor.h:161
static FPrimitiveAdded OnPrimitiveAdded
Definition: Sensor.h:354
FString GetSensorIdentifier() const
Definition: Sensor.h:74
virtual void CreateDataSavePath()
Definition: Sensor.cpp:229
FString GetSensorName() const
Definition: Sensor.h:95
void SetSimulateSensor(bool SimulateSensor)
Definition: Sensor.h:151
UTopic * GetROSTopic() const
Definition: Sensor.h:141
ULogFile * LogFile
Definition: Sensor.h:347
bool SendDataToROS
Definition: Sensor.h:344
static TArray< TWeakObjectPtr< UPrimitiveComponent > > GetComponentsToHide()
Definition: Sensor.h:256
FString FileSavePath
Definition: Sensor.h:349
void WriteToLogFile(const FString &Message)
Definition: Sensor.cpp:269
FORCEINLINE bool IsROSConnected() const
Definition: Sensor.h:192
virtual void CreateROSTopic()
Definition: Sensor.cpp:173
static void RemoveTick(FTickEntry TickEntry)
Definition: TickManager.cpp:90
static FTickEntry AddTick(UObject *Object, std::function< void(float)> Function, ETickType Type)
Definition: TickManager.cpp:55
bool IsVehicleInGarage() const
Definition: Vehicle.h:123
static void Log(const FString &Message, bool LogToTextFile=true, bool LogToROS=true)
virtual void Setup(ACamera *CameraRef, bool ShowOnStartup=true)
static FGeographicCoordinates UnrealToGeographicCoordinates(AGeoReferencingSystem *GeoReferencingSystem, const FVector &Position)
void Create(const FString &FileNameWithoutExtension, FLogFileSettings Settings)
Definition: LogFile.cpp:40
bool KeepFileOpen
Definition: LogFile.h:42
bool Timestamp
Definition: LogFile.h:39
FString FilePath
Definition: LogFile.h:54
FFileWriteOptions FileWriteOptions
Definition: LogFile.h:45
int32 QueueLength
Definition: LogFile.h:48
bool OverrideFilePath
Definition: LogFile.h:51
FFileCreationOptions FileCreationOptions
Definition: LogFile.h:36