16#include "NiagaraComponent.h"
17#include "Components/PrimitiveComponent.h"
18#include "Materials/Material.h"
19#include "ImageUtils.h"
20#include "Async/Async.h"
21#include "HighResScreenshot.h"
22#include "ImageWriteTask.h"
23#include "ImageWriteQueue.h"
24#include "Misc/DateTime.h"
25#include "Math/Color.h"
26#include "Blueprint/UserWidget.h"
27#include "GameFramework/PlayerController.h"
28#include "GeoReferencingSystem.h"
29#include "TimerManager.h"
31#include "GameFramework/SpringArmComponent.h"
34ACamera::ACamera(
const FObjectInitializer& ObjectInitializer) : Super(ObjectInitializer)
36 PrimaryActorTick.bCanEverTick =
true;
37 PrimaryActorTick.TickGroup = TG_PostUpdateWork;
39 CaptureRenderTarget = CreateDefaultSubobject<UTextureRenderTarget2D>(FName(*FString::Printf(TEXT(
"CaptureRenderTarget"))));
47 CaptureComponent2D = CreateDefaultSubobject<USceneCaptureComponent2D>(FName(*FString::Printf(TEXT(
"SceneCaptureComponent2D"))));
49 CaptureComponent2D->PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_RenderScenePrimitives;
62 UMaterial* PostProcessingMat = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
66 PostProcessSettings.AddBlendable(PostProcessingMat, Weight);
71 FString Msg =
"Failed to add Post processing material to " + Sensor;
81 PostProcessSettings.RemoveBlendable(Material);
109 LogFile = NewObject<ULogFile>(ULogFile::StaticClass());
112 FString FileName =
"camera_metadata";
119 WriteToLogFile(
"timestamp, image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation, GPS latitude, GPS longitude, GPS altitude");
123 WriteToLogFile(
"timestamp, image_name, X location, Y location, Z location, yaw rotation, pitch rotation, roll rotation");
133 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: CaptureComponent2D is nullptr!"));
154 const FString Path =
"/Game/Agrarsense/Materials/PostProcessingMaterials/PhysicLensDistortion.PhysicLensDistortion";
155 PhysicLensDistortion = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
161 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Added physics lens distortion effect."));
171 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Removed physics lens distortion effect."));
179 const FString Path =
"/Game/Agrarsense/Materials/PostProcessingMaterials/CameraPostProcessEffects/m_ice_lens_effect";
180 UMaterial* LoadedIceMaterial = Cast<UMaterial>(StaticLoadObject(UMaterial::StaticClass(),
nullptr, *Path));
189 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Added ice lens effect."));
199 IceMatInstance->SetScalarParameterValue(FName(
"Angle"), parameters.
IceLensEffectAngle);
201 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Changed ice lens effect parameters."));
211 PostProcessSettings.RemoveBlendable(IceMatInstance);
215 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Removed ice lens effect."));
230 PostProcessSettings.bOverride_AutoExposureMethod = UsePostProcessingEffects;
231 PostProcessSettings.bOverride_AutoExposureBias = UsePostProcessingEffects;
232 PostProcessSettings.bOverride_AutoExposureMinBrightness = UsePostProcessingEffects;
233 PostProcessSettings.bOverride_AutoExposureMaxBrightness = UsePostProcessingEffects;
234 PostProcessSettings.bOverride_AutoExposureSpeedUp = UsePostProcessingEffects;
235 PostProcessSettings.bOverride_AutoExposureSpeedDown = UsePostProcessingEffects;
236 PostProcessSettings.bOverride_HistogramLogMin = UsePostProcessingEffects;
237 PostProcessSettings.bOverride_HistogramLogMax = UsePostProcessingEffects;
238 PostProcessSettings.bOverride_CameraShutterSpeed = UsePostProcessingEffects;
239 PostProcessSettings.bOverride_CameraISO = UsePostProcessingEffects;
240 PostProcessSettings.bOverride_DepthOfFieldFstop = UsePostProcessingEffects;
241 PostProcessSettings.bOverride_DepthOfFieldMinFstop = UsePostProcessingEffects;
242 PostProcessSettings.bOverride_DepthOfFieldBladeCount = UsePostProcessingEffects;
243 PostProcessSettings.bOverride_FilmSlope = UsePostProcessingEffects;
244 PostProcessSettings.bOverride_FilmToe = UsePostProcessingEffects;
245 PostProcessSettings.bOverride_FilmShoulder = UsePostProcessingEffects;
246 PostProcessSettings.bOverride_FilmWhiteClip = UsePostProcessingEffects;
247 PostProcessSettings.bOverride_FilmBlackClip = UsePostProcessingEffects;
248 PostProcessSettings.bOverride_MotionBlurAmount = UsePostProcessingEffects;
249 PostProcessSettings.bOverride_MotionBlurMax = UsePostProcessingEffects;
250 PostProcessSettings.bOverride_MotionBlurPerObjectSize = UsePostProcessingEffects;
251 PostProcessSettings.bOverride_WhiteTemp = UsePostProcessingEffects;
252 PostProcessSettings.bOverride_WhiteTint = UsePostProcessingEffects;
253 PostProcessSettings.bOverride_ColorContrast = UsePostProcessingEffects;
254 PostProcessSettings.bOverride_SceneFringeIntensity = UsePostProcessingEffects;
255 PostProcessSettings.bOverride_ChromaticAberrationStartOffset = UsePostProcessingEffects;
256 PostProcessSettings.bOverride_AmbientOcclusionIntensity = UsePostProcessingEffects;
257 PostProcessSettings.bOverride_AmbientOcclusionRadius = UsePostProcessingEffects;
258 PostProcessSettings.bOverride_AmbientOcclusionStaticFraction = UsePostProcessingEffects;
259 PostProcessSettings.bOverride_AmbientOcclusionFadeDistance = UsePostProcessingEffects;
260 PostProcessSettings.bOverride_AmbientOcclusionPower = UsePostProcessingEffects;
261 PostProcessSettings.bOverride_AmbientOcclusionBias = UsePostProcessingEffects;
262 PostProcessSettings.bOverride_AmbientOcclusionQuality = UsePostProcessingEffects;
263 PostProcessSettings.bOverride_BloomMethod = UsePostProcessingEffects;
264 PostProcessSettings.bOverride_BloomIntensity = UsePostProcessingEffects;
265 PostProcessSettings.bOverride_BloomThreshold = UsePostProcessingEffects;
266 PostProcessSettings.bOverride_LensFlareIntensity = UsePostProcessingEffects;
267 PostProcessSettings.bOverride_DepthOfFieldFocalDistance = UsePostProcessingEffects;
268 PostProcessSettings.bOverride_DepthOfFieldDepthBlurAmount = UsePostProcessingEffects;
269 PostProcessSettings.bOverride_DepthOfFieldDepthBlurRadius = UsePostProcessingEffects;
327 FString Encoding =
"bgr8";
331 ImageMsg = MakeShared<ROSMessages::sensor_msgs::Image>();
332 ImageMsg->header.frame_id =
"world";
352 FSoftClassPath WidgetClassPath(TEXT(
"/Game/Agrarsense/Widgets/Camera/WBP_CameraControlsMenu.WBP_CameraControlsMenu_C"));
353 UClass* WidgetClass = WidgetClassPath.TryLoadClass<UUserWidget>();
355 APlayerController* PlayerController = UGameplayStatics::GetPlayerController(GetWorld(), 0);
357 if (WidgetClass && PlayerController)
359 UCameraWidgetBase* WidgetInstance = CreateWidget<UCameraWidgetBase>(PlayerController, WidgetClass);
362 bool ShowGuide =
false;
363 WidgetInstance->
Setup(
this, ShowGuide);
369 bool Resized =
false;
405 SetActorTickEnabled(
false);
410 SetActorTickEnabled(
true);
416 for (int32 i = 0; i < Components.Num(); i++)
418 UPrimitiveComponent* Primitive = Components[i].Get();
427 GetWorld()->GetTimerManager().SetTimerForNextTick([
this]()
431 DronePtr = Cast<APIDDrone>(GetAttachParentActor());
442 Super::EndPlay(EndPlayReason);
501 SetActorRotation(gimbalRot);
569 TArray<FCapturedFrameData> Frames =
FrameGrabber->GetCapturedFrames();
572 FCapturedFrameData& LastFrame = Frames.Last();
573 TArray<FColor>& ImageBuffer = LastFrame.ColorBuffer;
585 AsyncTask(ENamedThreads::AnyBackgroundThreadNormalTask, [
this, ImageBuffer]()
602 static uint8 GammaLUT[256];
603 static bool LUTInitialized =
false;
606 LUTInitialized =
true;
607 for (int32 i = 0; i < 256; ++i)
610 float Linear = FMath::Pow(i / 255.0f, 2.2f);
611 GammaLUT[i] = FMath::RoundToInt(Linear * 255.0f);
615 FColor* Ptr = Buffer.GetData();
616 int32 Num = Buffer.Num();
618 for (int32 i = 0; i < Num; ++i)
640 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: UnrealWindow is nullptr!"));
648 TSharedPtr<FSceneViewport> SceneViewport =
UnrealWindow->GetSceneViewport();
651 EPixelFormat PixelFormat = GetPixelFormatFromRenderTargetFormat(
TextureFormat);
652 FIntPoint Size = SceneViewport->GetSize();
661 FrameGrabber = MakeShareable(
new FFrameGrabber(SceneViewport.ToSharedRef(), Size, PixelFormat));
691 || FrameBuffer.IsEmpty()
708 const int32 BufferSize = Width * Height * 3;
719 const FColor* Source = FrameBuffer.GetData();
720 const int32 PixelCount = FrameBuffer.Num();
722 for (int32 i = 0; i < PixelCount; ++i)
724 const FColor& Pixel = Source[i];
737 ImageMsg->header.time = FROSTime::Now();
747 if (FrameBuffer.IsEmpty() || Width == 0 || Height == 0)
752 int32 Size = Width * Height;
753 if (FrameBuffer.Num() != Size)
755 FString Msg =
"Camera sensor: Unable to save Image to disk. FrameBuffer size and resolution don't match!";
760 FString ImageName = FString::Printf(TEXT(
"image_%d.png"),
ImageNumber);
761 FString fullFileName = FString::Printf(TEXT(
"%s%s"), *
FileSavePath, *ImageName);
766 FIntPoint DestSize(Width, Height);
767 TImagePixelData<FColor> PixelData(DestSize);
768 PixelData.Pixels = FrameBuffer;
771 TUniquePtr<FImageWriteTask> ImageTask = MakeUnique<FImageWriteTask>();
772 ImageTask->PixelData = MakeUnique<TImagePixelData<FColor>>(PixelData);
773 ImageTask->Filename = fullFileName;
774 ImageTask->Format = EImageFormat::PNG;
775 ImageTask->CompressionQuality = (int32)EImageCompressionQuality::Default;
776 ImageTask->bOverwriteFile =
true;
777 ImageTask->PixelPreProcessors.Add(TAsyncAlphaWrite<FColor>(255));
780 FHighResScreenshotConfig& HighResScreenshotConfig = GetHighResScreenshotConfig();
781 TFuture<bool> CompletionFuture = HighResScreenshotConfig.ImageWriteQueue->Enqueue(MoveTemp(ImageTask));
792 const FVector ActorPosition = GetActorLocation();
793 const FRotator ActorRotation = GetActorRotation();
802 MetaData = FString::Printf(TEXT(
"%s, %s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.8f, %.8f, %.8f"),
803 *TimeStamp, *ImageName,
804 ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
805 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll,
806 GeoCoordinates.Latitude, GeoCoordinates.Longitude, GeoCoordinates.Altitude);
810 MetaData = FString::Printf(TEXT(
"%s, %s, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f"),
811 *TimeStamp, *ImageName,
812 ActorPosition.X, ActorPosition.Y, ActorPosition.Z,
813 ActorRotation.Pitch, ActorRotation.Yaw, ActorRotation.Roll);
823 UE_LOG(LogTemp, Warning, TEXT(
"Camera.cpp: Camera window closed. Destroying Camera sensor.."));
854 CameraShowFlags.SetDynamicShadows(RenderShadows);
855 CameraShowFlags.SetContactShadows(RenderShadows);
856 CameraShowFlags.SetCapsuleShadows(RenderShadows);
865 CameraShowFlags.SetTemporalAA(SetTemporal);
871 if (Width == 0 || Height == 0)
898 ShowFlags.SetDeferredLighting(
false);
899 ShowFlags.SetSkyLighting(
false);
900 ShowFlags.SetAmbientOcclusion(
false);
901 ShowFlags.SetAntiAliasing(
false);
902 ShowFlags.SetVolumetricFog(
false);;
903 ShowFlags.SetBloom(
false);
904 ShowFlags.SetCameraImperfections(
false);
905 ShowFlags.SetCameraInterpolation(
false);
906 ShowFlags.SetColorGrading(
false);
907 ShowFlags.SetDepthOfField(
false);
908 ShowFlags.SetDiffuse(
false);
909 ShowFlags.SetDirectionalLights(
false);
910 ShowFlags.SetDirectLighting(
false);
911 ShowFlags.SetDynamicShadows(
false);
912 ShowFlags.SetEyeAdaptation(
false);
913 ShowFlags.SetFog(
false);
914 ShowFlags.SetGlobalIllumination(
false);
915 ShowFlags.SetGrain(
false);
916 ShowFlags.SetHLODColoration(
false);
917 ShowFlags.SetHMDDistortion(
false);
918 ShowFlags.SetLensFlares(
false);
919 ShowFlags.SetLightComplexity(
false);
920 ShowFlags.SetLightFunctions(
false);
921 ShowFlags.SetLightInfluences(
false);
922 ShowFlags.SetLighting(
false);
923 ShowFlags.SetLightMapDensity(
false);
924 ShowFlags.SetLightRadius(
false);
925 ShowFlags.SetLightShafts(
false);
926 ShowFlags.SetLODColoration(
false);
927 ShowFlags.SetMotionBlur(
false);
928 ShowFlags.SetOnScreenDebug(
false);
929 ShowFlags.SetParticles(
false);
930 ShowFlags.SetPointLights(
false);
931 ShowFlags.SetRefraction(
false);
932 ShowFlags.SetSceneColorFringe(
false);
933 ShowFlags.SetScreenSpaceAO(
false);
934 ShowFlags.SetScreenSpaceReflections(
false);
935 ShowFlags.SetSkyLighting(
false);
936 ShowFlags.SetSpotLights(
false);
937 ShowFlags.SetStationaryLightOverlap(
false);
938 ShowFlags.SetSubsurfaceScattering(
false);
939 ShowFlags.SetTonemapper(
false);
940 ShowFlags.SetVisualizeBuffer(
false);
941 ShowFlags.SetVisualizeDistanceFieldAO(
false);
942 ShowFlags.SetVisualizeDOF(
false);
943 ShowFlags.SetVisualizeHDR(
false);
944 ShowFlags.SetVisualizeLightCulling(
false);
945 ShowFlags.SetVisualizeMeshDistanceFields(
false);
946 ShowFlags.SetVisualizeMotionBlur(
false);
947 ShowFlags.SetVisualizeOutOfBoundsPixels(
false);
948 ShowFlags.SetVisualizeSenses(
false);
949 ShowFlags.SetVisualizeShadingModels(
false);
950 ShowFlags.SetVisualizeSSR(
false);
951 ShowFlags.SetVisualizeSSS(
false);
static auto BindTick(ObjectType *Object, FunctionType Function)
void AddPostProcessingMaterial(const FString &Path, float Weight=1.0f)
TSharedPtr< FFrameGrabber > FrameGrabber
void ChangeCameraParameters(FCameraBaseParameters newParameters)
virtual ESensorTypes GetSensorType() const override
FCameraBaseParameters CameraParameters
void ApplyGammaCorrectionLUT(TArray< FColor > &Buffer)
FOnWindowClosed OnWindowClosedDelegate
void AddHiddenActor(AActor *Actor)
virtual void Init(FCameraBaseParameters parameters, bool SimulateSensor=true)
virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override
USceneCaptureComponent2D * CaptureComponent2D
void RemovePostProcessingMaterial(UMaterial *Material)
void SaveCameraMetaDataToDisk(const FString &ImageName)
AGeoReferencingSystem * GeoReferencingSystem
void ResizeCamera(int Width=1280, int Height=720)
void SaveImageToDisk(const TArray< FColor > FrameBuffer, int32 Width, int32 Height)
FDelegateHandle OnPostTickDelegate
bool ShouldSimulateCamera
void SetShadowRendering(bool RenderShadows)
void OnWindowClosed(const TSharedRef< SWindow > &Window)
void FrameGrabberCapture()
void SendImageDataToROS(const TArray< FColor > &FrameBuffer, int32 Width, int32 Height)
FDelegateHandle OnPreTickDelegate
UTextureRenderTarget2D * CaptureRenderTarget
void AddWidgetToWindow(UWidget *WidgetToAdd)
TSharedPtr< ROSMessages::sensor_msgs::Image > ImageMsg
FRotator SensorSetRotation
bool SaveCurrentFrameToDiskRequested
void SetTemporalAA(bool SetTemporal)
void CreateLogFile() override
UTexture2D * CaptureFrameTexture
void SetupCamera(FCameraBaseParameters parameters)
TWeakObjectPtr< UMaterial > PhysicLensDistortion
void RemoveWidgetFromWindow(UWidget *WidgetToRemove)
FCameraDelegate_OnWindowResized OnCameraWindowResized
void HidePrimitiveComponent(UPrimitiveComponent *PrimitiveComponent)
TEnumAsByte< ETextureRenderTargetFormat > TextureFormat
void ReleaseFrameGrabber()
void EndOfFrameParellel(float DeltaTime)
virtual void AddProcessingToFrameBuffer(TArray< FColor > &buffer)
TSharedPtr< FUnrealWindow > UnrealWindow
void EndOfFrame(UWorld *World, ELevelTick TickType, float DeltaSeconds)
ACamera(const FObjectInitializer &ObjectInitializer)
void PreActorTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
FCameraDelegate_OnWindowClosed OnCameraWindowClosed
virtual void BeginPlay() override
TWeakObjectPtr< UMaterialInstanceDynamic > IceMaterialInstance
bool ShouldSimulate(const float DeltaSeconds)
bool CanSimulateSensor() const
static FPrimitiveAdded OnPrimitiveAdded
FString GetSensorIdentifier() const
virtual void CreateDataSavePath()
FString GetSensorName() const
void SetSimulateSensor(bool SimulateSensor)
UTopic * GetROSTopic() const
static TArray< TWeakObjectPtr< UPrimitiveComponent > > GetComponentsToHide()
FString CreateTimeStampString() const
void WriteToLogFile(const FString &Message)
FORCEINLINE bool IsROSConnected() const
virtual void CreateROSTopic()
static void RemoveTick(FTickEntry TickEntry)
static FTickEntry AddTick(UObject *Object, std::function< void(float)> Function, ETickType Type)
bool IsVehicleInGarage() const
static void Log(const FString &Message, bool LogToTextFile=true, bool LogToROS=true)
static bool ShouldStartWindowMinimized()
static FGeographicCoordinates UnrealToGeographicCoordinates(AGeoReferencingSystem *GeoReferencingSystem, const FVector &Position)
void Create(const FString &FileNameWithoutExtension, FLogFileSettings Settings)
bool UsePhysicLensDistortionEffect
float MaxViewDistanceInCmOverride
float ExposureMaxBrightness
float IceLensEffectStrength
float MotionBlurMinObjSize
float ExposureMinBrightness
float MotionBlurIntensity
bool PostProcessingEffects
float ChromAberrIntensity
FFileWriteOptions FileWriteOptions
FFileCreationOptions FileCreationOptions