17#include "NiagaraComponent.h"
18#include "Components/PrimitiveComponent.h"
19#include "Materials/Material.h"
20#include "ImageUtils.h"
21#include "Async/Async.h"
22#include "HighResScreenshot.h"
23#include "ImageWriteTask.h"
24#include "ImageWriteQueue.h"
25#include "Misc/DateTime.h"
26#include "Math/Color.h"
27#include "Blueprint/UserWidget.h"
28#include "GameFramework/PlayerController.h"
29#include "GeoReferencingSystem.h"
30#include "TimerManager.h"
31#include "GameFramework/SpringArmComponent.h"
32#include "Async/ParallelFor.h"
34ACamera::ACamera(
const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer)
36 PrimaryActorTick.bCanEverTick =
true;
37 PrimaryActorTick.TickGroup = TG_PostUpdateWork;
39 CaptureRenderTarget = CreateDefaultSubobject<UTextureRenderTarget2D>(FName(*FString::Printf(TEXT(
"CaptureRenderTarget"))));
49 CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(FName(*FString::Printf(TEXT(
"SceneCaptureComponent2D"))));
51 CaptureComponent2D->PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_RenderScenePrimitives;
64 UMaterial* PostProcessingMat = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
68 PostProcessSettings.AddBlendable(PostProcessingMat, Weight);
72 const FString ErrorMessage = FString::Printf(TEXT(
"Failed to add post-processing material to sensor: %s"), *
GetSensorName());
82 PostProcessSettings.RemoveBlendable(Material);
110 LogFile = NewObject<ULogFile>(ULogFile::StaticClass());
113 const FString FileName =
"camera_metadata";
120 WriteToLogFile(
"timestamp, image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation, GPS latitude, GPS longitude, GPS altitude");
124 WriteToLogFile(
"timestamp, image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation");
134 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: CaptureComponent2D is nullptr!"));
155 const FString Path =
"/Game/Agrarsense/Materials/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion";
156 PhysicLensDistortion = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
162 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Added physics lens distortion effect."));
172 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Removed physics lens distortion effect."));
180 const FString Path =
"/Game/Agrarsense/Materials/PostProcessingMaterials/CameraPostProcessEffects/m_ice_lens_effect";
181 UMaterial* LoadedIceMaterial = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
190 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Added ice lens effect."));
200 IceMatInstance->SetScalarParameterValue(FName(
"Angle"), parameters.
IceLensEffectAngle);
202 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Changed ice lens effect parameters."));
212 PostProcessSettings.RemoveBlendable(IceMatInstance);
216 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Removed ice lens effect."));
228 EPixelFormat PixelFormat = Enable16BitFormat ? PF_FloatRGB : PF_B8G8R8A8;
232 PixelFormat = PF_B8G8R8A8;
233 ForceLinearGamma =
true;
242 PostProcessSettings.bOverride_AutoExposureMethod = UsePostProcessingEffects;
243 PostProcessSettings.bOverride_AutoExposureBias = UsePostProcessingEffects;
244 PostProcessSettings.bOverride_AutoExposureMinBrightness = UsePostProcessingEffects;
245 PostProcessSettings.bOverride_AutoExposureMaxBrightness = UsePostProcessingEffects;
246 PostProcessSettings.bOverride_AutoExposureSpeedUp = UsePostProcessingEffects;
247 PostProcessSettings.bOverride_AutoExposureSpeedDown = UsePostProcessingEffects;
248 PostProcessSettings.bOverride_HistogramLogMin = UsePostProcessingEffects;
249 PostProcessSettings.bOverride_HistogramLogMax = UsePostProcessingEffects;
250 PostProcessSettings.bOverride_CameraShutterSpeed = UsePostProcessingEffects;
251 PostProcessSettings.bOverride_CameraISO = UsePostProcessingEffects;
252 PostProcessSettings.bOverride_DepthOfFieldFstop = UsePostProcessingEffects;
253 PostProcessSettings.bOverride_DepthOfFieldMinFstop = UsePostProcessingEffects;
254 PostProcessSettings.bOverride_DepthOfFieldBladeCount = UsePostProcessingEffects;
255 PostProcessSettings.bOverride_FilmSlope = UsePostProcessingEffects;
256 PostProcessSettings.bOverride_FilmToe = UsePostProcessingEffects;
257 PostProcessSettings.bOverride_FilmShoulder = UsePostProcessingEffects;
258 PostProcessSettings.bOverride_FilmWhiteClip = UsePostProcessingEffects;
259 PostProcessSettings.bOverride_FilmBlackClip = UsePostProcessingEffects;
260 PostProcessSettings.bOverride_MotionBlurAmount = UsePostProcessingEffects;
261 PostProcessSettings.bOverride_MotionBlurMax = UsePostProcessingEffects;
262 PostProcessSettings.bOverride_MotionBlurPerObjectSize = UsePostProcessingEffects;
263 PostProcessSettings.bOverride_WhiteTemp = UsePostProcessingEffects;
264 PostProcessSettings.bOverride_WhiteTint = UsePostProcessingEffects;
265 PostProcessSettings.bOverride_ColorContrast = UsePostProcessingEffects;
266 PostProcessSettings.bOverride_SceneFringeIntensity = UsePostProcessingEffects;
267 PostProcessSettings.bOverride_ChromaticAberrationStartOffset = UsePostProcessingEffects;
268 PostProcessSettings.bOverride_AmbientOcclusionIntensity = UsePostProcessingEffects;
269 PostProcessSettings.bOverride_AmbientOcclusionRadius = UsePostProcessingEffects;
270 PostProcessSettings.bOverride_AmbientOcclusionStaticFraction = UsePostProcessingEffects;
271 PostProcessSettings.bOverride_AmbientOcclusionFadeDistance = UsePostProcessingEffects;
272 PostProcessSettings.bOverride_AmbientOcclusionPower = UsePostProcessingEffects;
273 PostProcessSettings.bOverride_AmbientOcclusionBias = UsePostProcessingEffects;
274 PostProcessSettings.bOverride_AmbientOcclusionQuality = UsePostProcessingEffects;
275 PostProcessSettings.bOverride_BloomMethod = UsePostProcessingEffects;
276 PostProcessSettings.bOverride_BloomIntensity = UsePostProcessingEffects;
277 PostProcessSettings.bOverride_BloomThreshold = UsePostProcessingEffects;
278 PostProcessSettings.bOverride_LensFlareIntensity = UsePostProcessingEffects;
279 PostProcessSettings.bOverride_DepthOfFieldFocalDistance = UsePostProcessingEffects;
280 PostProcessSettings.bOverride_DepthOfFieldDepthBlurAmount = UsePostProcessingEffects;
281 PostProcessSettings.bOverride_DepthOfFieldDepthBlurRadius = UsePostProcessingEffects;
339 const FString Encoding =
"bgr8";
343 ImageMsg = MakeShared<ROSMessages::sensor_msgs::Image>();
344 ImageMsg->header.frame_id =
"world";
364 FSoftClassPath WidgetClassPath(TEXT(
"/Game/Agrarsense/Widgets/Camera/WBP_CameraControlsMenu.WBP_CameraControlsMenu_C"));
365 UClass* WidgetClass = WidgetClassPath.TryLoadClass<UUserWidget>();
367 APlayerController* PlayerController = UGameplayStatics::GetPlayerController(GetWorld(), 0);
369 if (WidgetClass && PlayerController)
371 UCameraWidgetBase* WidgetInstance = CreateWidget<UCameraWidgetBase>(PlayerController, WidgetClass);
374 bool ShowGuide =
false;
375 WidgetInstance->
Setup(
this, ShowGuide);
381 bool Resized =
false;
417 SetActorTickEnabled(
false);
422 SetActorTickEnabled(
true);
426 UE_LOG(LogTemp, Warning, TEXT(
"juu"));
437 for (int32 i = 0; i < Components.Num(); i++)
439 UPrimitiveComponent* Primitive = Components[i].Get();
448 GetWorld()->GetTimerManager().SetTimerForNextTick([
this]()
452 DronePtr = Cast<APIDDrone>(GetAttachParentActor());
463 Super::EndPlay(EndPlayReason);
541 SetActorRotation(GimbalRot);
601 TArray<FCapturedFrameData> Frames =
FrameGrabber->GetCapturedFrames();
604 FCapturedFrameData& LastFrame = Frames.Last();
605 TArray<FColor>& ImageBuffer = LastFrame.ColorBuffer;
617 AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [
this, ImageBuffer]()
641 TArray<FColor> ImageBuffer;
643 FReadSurfaceDataFlags Flags(RCM_UNorm);
644 Flags.SetLinearToGamma(
false);
646 const bool succeeded = Resource->ReadPixels(ImageBuffer, Flags);
656 AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [
this, ImageBuffer]()
672 static uint8 GammaLUT[256];
673 static bool LUTInitialized =
false;
676 LUTInitialized =
true;
677 for (int32 i = 0; i < 256; ++i)
679 float Linear = FMath::Pow(i / 255.0f, 2.2f);
680 GammaLUT[i] = FMath::RoundToInt(Linear * 255.0f);
684 FColor* RESTRICT BufferPtr = Buffer.GetData();
685 const int32 Num = Buffer.Num();
687 const int32 ChunkSize = 512;
688 const int32 NumChunks = (Num + ChunkSize - 1) / ChunkSize;
690 ParallelFor(NumChunks, [BufferPtr, Num](int32 ChunkIndex)
692 const int32 Start = ChunkIndex * ChunkSize;
693 const int32 End = FMath::Min(Start + ChunkSize, Num);
695 for (int32 i = Start; i < End; ++i)
697 FColor& C = BufferPtr[i];
718 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: UnrealWindow is nullptr!"));
726 TSharedPtr<FSceneViewport> SceneViewport =
UnrealWindow->GetSceneViewport();
729 EPixelFormat PixelFormat = GetPixelFormatFromRenderTargetFormat(
TextureFormat);
730 FIntPoint Size = SceneViewport->GetSize();
739 FrameGrabber = MakeShareable(
new FFrameGrabber(SceneViewport.ToSharedRef(), Size, PixelFormat));
769 || FrameBuffer.IsEmpty()
785 const int32 BufferSize = Width * Height * 3;
794 const FColor* RESTRICT Source = FrameBuffer.GetData();
796 const int32 PixelCount = FrameBuffer.Num();
797 const int32 ChunkSize = 512;
798 const int32 NumChunks = (PixelCount + ChunkSize - 1) / ChunkSize;
800 ParallelFor(NumChunks, [Source, Dest, ChunkSize, PixelCount](int32 ChunkIndex)
802 const int32 StartIndex = ChunkIndex * ChunkSize;
803 const int32 EndIndex = FMath::Min(StartIndex + ChunkSize, PixelCount);
805 for (int32 i = StartIndex; i < EndIndex; ++i)
807 const FColor& Pixel = Source[i];
808 const int32 j = i * 3;
809 Dest[j + 0] = Pixel.B;
810 Dest[j + 1] = Pixel.G;
811 Dest[j + 2] = Pixel.R;
820 ImageMsg->header.time = FROSTime::Now();
829 if (FrameBuffer.IsEmpty() || Width == 0 || Height == 0)
834 int32 Size = Width * Height;
835 if (FrameBuffer.Num() != Size)
837 const FString Msg =
"Camera sensor: Unable to save Image to disk. FrameBuffer size and resolution don't match!";
842 const FString ImageName = FString::Printf(TEXT(
"image_%d.png"),
ImageNumber);
843 const FString FullFileName = FString::Printf(TEXT(
"%s%s"), *
FileSavePath, *ImageName);
848 FIntPoint DestSize(Width, Height);
849 TImagePixelData<FColor> PixelData(DestSize);
850 PixelData.Pixels = FrameBuffer;
853 TUniquePtr<FImageWriteTask> ImageTask = MakeUnique<FImageWriteTask>();
854 ImageTask->PixelData = MakeUnique<TImagePixelData<FColor>>(PixelData);
855 ImageTask->Filename = FullFileName;
856 ImageTask->Format = EImageFormat::PNG;
857 ImageTask->CompressionQuality = (int32)EImageCompressionQuality::Default;
858 ImageTask->bOverwriteFile =
true;
859 ImageTask->PixelPreProcessors.Add(TAsyncAlphaWrite<FColor>(255));
862 FHighResScreenshotConfig& HighResScreenshotConfig = GetHighResScreenshotConfig();
863 TFuture<bool> CompletionFuture = HighResScreenshotConfig.ImageWriteQueue->Enqueue(MoveTemp(ImageTask));
874 const FVector ActorPosition = GetActorLocation();
875 const FRotator ActorRotation = GetActorRotation();
883 MetaData = FString::Printf(TEXT(
"%s, %s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.8f, %.8f, %.8f"),
884 *TimeStamp, *ImageName,
885 ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
886 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll,
887 GeoCoordinates.Latitude, GeoCoordinates.Longitude, GeoCoordinates.Altitude);
891 MetaData = FString::Printf(TEXT(
"%s, %s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f"),
892 *TimeStamp, *ImageName,
893 ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
894 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll);
904 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Camera window closed. Destroying Camera sensor.."));
935 CameraShowFlags.SetDynamicShadows(RenderShadows);
936 CameraShowFlags.SetContactShadows(RenderShadows);
937 CameraShowFlags.SetCapsuleShadows(RenderShadows);
946 CameraShowFlags.SetTemporalAA(SetTemporal);
952 if (Width == 0 || Height == 0)
979 ShowFlags.SetDeferredLighting(
false);
980 ShowFlags.SetSkyLighting(
false);
981 ShowFlags.SetAmbientOcclusion(
false);
982 ShowFlags.SetAntiAliasing(
false);
983 ShowFlags.SetVolumetricFog(
false);;
984 ShowFlags.SetBloom(
false);
985 ShowFlags.SetCameraImperfections(
false);
986 ShowFlags.SetCameraInterpolation(
false);
987 ShowFlags.SetColorGrading(
false);
988 ShowFlags.SetDepthOfField(
false);
989 ShowFlags.SetDiffuse(
false);
990 ShowFlags.SetDirectionalLights(
false);
991 ShowFlags.SetDirectLighting(
false);
992 ShowFlags.SetDynamicShadows(
false);
993 ShowFlags.SetEyeAdaptation(
false);
994 ShowFlags.SetFog(
false);
995 ShowFlags.SetGlobalIllumination(
false);
996 ShowFlags.SetGrain(
false);
997 ShowFlags.SetHLODColoration(
false);
998 ShowFlags.SetHMDDistortion(
false);
999 ShowFlags.SetLensFlares(
false);
1000 ShowFlags.SetLightComplexity(
false);
1001 ShowFlags.SetLightFunctions(
false);
1002 ShowFlags.SetLightInfluences(
false);
1003 ShowFlags.SetLighting(
false);
1004 ShowFlags.SetLightMapDensity(
false);
1005 ShowFlags.SetLightRadius(
false);
1006 ShowFlags.SetLightShafts(
false);
1007 ShowFlags.SetLODColoration(
false);
1008 ShowFlags.SetMotionBlur(
false);
1009 ShowFlags.SetOnScreenDebug(
false);
1010 ShowFlags.SetParticles(
false);
1011 ShowFlags.SetPointLights(
false);
1012 ShowFlags.SetRefraction(
false);
1013 ShowFlags.SetSceneColorFringe(
false);
1014 ShowFlags.SetScreenSpaceAO(
false);
1015 ShowFlags.SetScreenSpaceReflections(
false);
1016 ShowFlags.SetSkyLighting(
false);
1017 ShowFlags.SetSpotLights(
false);
1018 ShowFlags.SetStationaryLightOverlap(
false);
1019 ShowFlags.SetSubsurfaceScattering(
false);
1020 ShowFlags.SetTonemapper(
false);
1021 ShowFlags.SetVisualizeBuffer(
false);
1022 ShowFlags.SetVisualizeDistanceFieldAO(
false);
1023 ShowFlags.SetVisualizeDOF(
false);
1024 ShowFlags.SetVisualizeHDR(
false);
1025 ShowFlags.SetVisualizeLightCulling(
false);
1026 ShowFlags.SetVisualizeMeshDistanceFields(
false);
1027 ShowFlags.SetVisualizeMotionBlur(
false);
1028 ShowFlags.SetVisualizeOutOfBoundsPixels(
false);
1029 ShowFlags.SetVisualizeSenses(
false);
1030 ShowFlags.SetVisualizeShadingModels(
false);
1031 ShowFlags.SetVisualizeSSR(
false);
1032 ShowFlags.SetVisualizeSSS(
false);
@ InstanceSegmentationCamera
static auto BindTick(ObjectType *Object, FunctionType Function)
void UpdateCameraStabilization()
void AddPostProcessingMaterial(const FString &Path, float Weight=1.0f)
TSharedPtr< FFrameGrabber > FrameGrabber
void ChangeCameraParameters(FCameraBaseParameters newParameters)
virtual ESensorTypes GetSensorType() const override
FCameraBaseParameters CameraParameters
void ApplyGammaCorrectionLUT(TArray< FColor > &Buffer)
FOnWindowClosed OnWindowClosedDelegate
void AddHiddenActor(AActor *Actor)
virtual void Init(FCameraBaseParameters parameters, bool SimulateSensor=true)
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override
USceneCaptureComponent2D * CaptureComponent2D
void RemovePostProcessingMaterial(UMaterial *Material)
void SaveCameraMetaDataToDisk(const FString &ImageName)
AGeoReferencingSystem * GeoReferencingSystem
void SaveImageToDisk(const TArray< FColor > FrameBuffer, int32 Width, int32 Height)
FDelegateHandle OnPostTickDelegate
bool ShouldSimulateCamera
void PixelReadEndOfFrame(UWorld *World, ELevelTick TickType, float DeltaSeconds)
void SetShadowRendering(bool RenderShadows)
void OnWindowClosed(const TSharedRef< SWindow > &Window)
void FrameGrabberCapture()
void SendImageDataToROS(const TArray< FColor > &FrameBuffer, int32 Width, int32 Height)
FDelegateHandle OnPreTickDelegate
UTextureRenderTarget2D * CaptureRenderTarget
void AddWidgetToWindow(UWidget *WidgetToAdd)
TSharedPtr< ROSMessages::sensor_msgs::Image > ImageMsg
FRotator SensorSetRotation
bool SaveCurrentFrameToDiskRequested
void SetTemporalAA(bool SetTemporal)
void CreateLogFile() override
UTexture2D * CaptureFrameTexture
void SetupCamera(FCameraBaseParameters parameters)
TWeakObjectPtr< UMaterial > PhysicLensDistortion
void RemoveWidgetFromWindow(UWidget *WidgetToRemove)
FCameraDelegate_OnWindowResized OnCameraWindowResized
void HidePrimitiveComponent(UPrimitiveComponent *PrimitiveComponent)
TEnumAsByte< ETextureRenderTargetFormat > TextureFormat
void ReleaseFrameGrabber()
void EndOfFrameParellel(float DeltaTime)
virtual void AddProcessingToFrameBuffer(TArray< FColor > &buffer)
void ResizeCamera(int32 Width=1280, int32 Height=720)
TSharedPtr< FUnrealWindow > UnrealWindow
void EndOfFrame(UWorld *World, ELevelTick TickType, float DeltaSeconds)
ACamera(const FObjectInitializer &ObjectInitializer)
void PreActorTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
FCameraDelegate_OnWindowClosed OnCameraWindowClosed
virtual void BeginPlay() override
TWeakObjectPtr< UMaterialInstanceDynamic > IceMaterialInstance
bool ShouldSimulate(const float DeltaSeconds)
bool CanSimulateSensor() const
static FPrimitiveAdded OnPrimitiveAdded
FString GetSensorIdentifier() const
virtual void CreateDataSavePath()
FString GetSensorName() const
void SetSimulateSensor(bool SimulateSensor)
UTopic * GetROSTopic() const
static TArray< TWeakObjectPtr< UPrimitiveComponent > > GetComponentsToHide()
FString CreateTimeStampString() const
void WriteToLogFile(const FString &Message)
FORCEINLINE bool IsROSConnected() const
virtual void CreateROSTopic()
static void RemoveTick(FTickEntry TickEntry)
static FTickEntry AddTick(UObject *Object, std::function< void(float)> Function, ETickType Type)
bool IsVehicleInGarage() const
static void Log(const FString &Message, bool LogToTextFile=true, bool LogToROS=true)
static bool ShouldStartWindowMinimized()
static FGeographicCoordinates UnrealToGeographicCoordinates(AGeoReferencingSystem *GeoReferencingSystem, const FVector &Position)
void Create(const FString &FileNameWithoutExtension, FLogFileSettings Settings)
bool UsePhysicLensDistortionEffect
float MaxViewDistanceInCmOverride
float ExposureMaxBrightness
float IceLensEffectStrength
float MotionBlurMinObjSize
float ExposureMinBrightness
float MotionBlurIntensity
bool PostProcessingEffects
float ChromAberrIntensity
FFileWriteOptions FileWriteOptions
FFileCreationOptions FileCreationOptions