Camera.h

Go to the documentation of this file.

// Copyright (c) 2023 FrostBit Software Lab at the Lapland University of Applied Sciences
//
// This work is licensed under the terms of the MIT license.
// For a copy, see <https://opensource.org/licenses/MIT>.
 
#pragma once
 
#include "CoreMinimal.h"
#include "Agrarsense/Sensor/Sensor.h"
 
#include "Engine/TextureRenderTarget2D.h"
#include "Kismet/KismetSystemLibrary.h"
#include "Delegates/IDelegateInstance.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "Components/SceneCaptureComponent2D.h"
#include "Engine/Texture2D.h"
#include "FrameGrabber.h"
#include "Materials/Material.h"
 
#include "Agrarsense/Sensor/Camera/CameraBaseParameters.h"
#include "Agrarsense/UnrealWindow/UnrealWindow.h"
#include "Agrarsense/Managers/TickManager.h"
 
#include "ROSIntegration/Public/sensor_msgs/Image.h"
 
#include "Camera.generated.h"
 
class AGeoReferencingSystem;
class UROSHandler;
class UWidget;
class APIDDrone;
 
DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FCameraDelegate_OnWindowClosed, ACamera*, Camera);
 
DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FCameraDelegate_OnWindowResized, ACamera*, Camera, FCameraBaseParameters, Params);
 
UCLASS()
class AGRARSENSE_API ACamera : public ASensor
{
    GENERATED_BODY()
 
public:
 
    ACamera(const FObjectInitializer& ObjectInitializer);
 
    virtual void Init(FCameraBaseParameters parameters, bool SimulateSensor = true);
 
    virtual ESensorTypes GetSensorType() const override 
    { 
        return ESensorTypes::RGBCamera; 
    }
 
    UFUNCTION(BlueprintCallable)
    void ChangeCameraParameters(FCameraBaseParameters newParameters);
 
    UFUNCTION(BlueprintCallable, BlueprintPure)
    FCameraBaseParameters GetCameraParameters()
    {
        return CameraParameters;
    }
 
    void AddPostProcessingMaterial(const FString& Path, float Weight = 1.0f);
 
    void RemovePostProcessingMaterial(UMaterial* Material);
 
    UFUNCTION(BlueprintCallable)
    USceneCaptureComponent2D* GetCaptureComponent2D() const
    {
        return CaptureComponent2D;
    }
 
    UFUNCTION(BlueprintCallable)
    UTextureRenderTarget2D* GetCaptureRenderTarget2D() const
    {
        return CaptureRenderTarget;
    }
 
    UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "FrameGrabber")
    TEnumAsByte<ETextureRenderTargetFormat> TextureFormat = ETextureRenderTargetFormat::RTF_RGBA8;
 
    UPROPERTY(BlueprintAssignable)
    FCameraDelegate_OnWindowClosed OnCameraWindowClosed;
 
    UPROPERTY(BlueprintAssignable)
    FCameraDelegate_OnWindowResized OnCameraWindowResized;
 
    UFUNCTION(BlueprintCallable)
    void AddWidgetToWindow(UWidget* WidgetToAdd);
 
    UFUNCTION(BlueprintCallable)
    void RemoveWidgetFromWindow(UWidget* WidgetToRemove);
 
    UFUNCTION(BlueprintCallable)
    void SetShadowRendering(bool RenderShadows);
 
    /*
    * Disables many camera show flags for faster rendering. Used for Depth/Semantic cameras.
    */
    void DisableShowFlags();
 
    UFUNCTION(BlueprintCallable)
    void SetTemporalAA(bool SetTemporal);
 
    UFUNCTION(BlueprintCallable)
    void ResizeCamera(int32 Width = 1280, int32 Height = 720);
 
    UFUNCTION(BlueprintCallable)
    void AddHiddenActor(AActor* Actor);
 
    UFUNCTION(BlueprintCallable)
    void SaveCurrentFrameToDisk()
    {
        if (!CameraParameters.SaveImageToDisk) 
        {
            SaveCurrentFrameToDiskRequested = true;
        }
    }
 
    UFUNCTION(BlueprintCallable)
    int32 GetCameraWidth() const
    {
        return ImageWidth;
    }
 
    UFUNCTION(BlueprintCallable)
    int32 GetCameraHeight() const
    {
        return ImageHeight;
    }
 
    UFUNCTION(BlueprintCallable)
    void SetUseGimbal(bool UseGimbal)
    {
        AllowGimbal = UseGimbal;
    }
 
    virtual FString GetParametersAsString() const override
    {
        return StructToString(CameraParameters);
    }
 
protected:
 
    virtual void BeginPlay() override;
 
    virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
 
    /*
    * Should we use ATicMananger to tick in parallel or use post actor tick
    */
    bool UseParallelLateTick = true;
 
    FDelegateHandle OnPreTickDelegate;
    void PreActorTick(UWorld* World, ELevelTick TickType, float DeltaSeconds);
 
    void UpdateCameraStabilization();
 
    FDelegateHandle OnPostTickDelegate;
    void EndOfFrame(UWorld* World, ELevelTick TickType, float DeltaSeconds);
    void PixelReadEndOfFrame(UWorld* World, ELevelTick TickType, float DeltaSeconds);
 
 
    void EndOfFrameParellel(float DeltaTime);
 
    /*
    *  Override function to all other camera variants if buffer editing is needed
    */
    virtual void AddProcessingToFrameBuffer(TArray<FColor>& buffer);
 
    void ApplyGammaCorrectionLUT(TArray<FColor>& Buffer);
 
    void SendImageDataToROS(const TArray<FColor>& FrameBuffer, int32 Width, int32 Height);
 
    UFUNCTION()
    void HidePrimitiveComponent(UPrimitiveComponent* PrimitiveComponent);
 
    void SaveImageToDisk(const TArray<FColor> FrameBuffer, int32 Width, int32 Height);
 
    void SaveCameraMetaDataToDisk(const FString& ImageName);
 
    void CreateLogFile() override;
 
    FString CameraName = "Camera ";
 
    FString FilePrefix = "Data/Camera_";
 
    UPROPERTY()
    UTextureRenderTarget2D* CaptureRenderTarget = nullptr;
 
    UPROPERTY(VisibleAnywhere)
    USceneCaptureComponent2D* CaptureComponent2D = nullptr;
 
    UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "FrameGrabber")
    UTexture2D* CaptureFrameTexture;
 
    TSharedPtr<FUnrealWindow> UnrealWindow;
 
    FTickEntry TickEntry;
 
    TWeakObjectPtr<UMaterialInstanceDynamic> IceMaterialInstance;
 
    TWeakObjectPtr<UMaterial> PhysicLensDistortion;
 
    TSharedPtr<ROSMessages::sensor_msgs::Image> ImageMsg;
 
    bool UseReadPixels = false;
 
    // For Gimbal
    APIDDrone* DronePtr = nullptr;
    FRotator SensorSetRotation;
    bool AllowGimbal = true;
    
private:
 
    void StartFrameGrabber();
 
    void ReleaseFrameGrabber();
 
    void FrameGrabberCapture();
 
    /*
    * Read pixels from CaptureRenderTarget.
    */
    void ReadPixelsCapture();
 
    void OnWindowClosed(const TSharedRef<SWindow>& Window);
 
    TSharedPtr<FFrameGrabber>FrameGrabber;
 
    void SetupCamera(FCameraBaseParameters parameters);
 
    bool ShouldSimulate(const float DeltaSeconds);
 
    UPROPERTY(VisibleAnywhere, Category = "Sensor")
    FCameraBaseParameters CameraParameters;
 
    /*
    * Temperaory Camera parameters. 
    * Camera parameter changes are applied at the end of the frame.
    */
    FCameraBaseParameters TempParams;
 
    UPROPERTY()
    AGeoReferencingSystem* GeoReferencingSystem = nullptr;
 
    bool ParametersChanged = false;
 
    FOnWindowClosed OnWindowClosedDelegate;
 
    int32 ImageWidth = 1280;
    int32 ImageHeight = 720;
 
    bool SaveCurrentFrameToDiskRequested = false;
 
    bool ShouldSimulateCamera = true;
 
    float CameraFrameRate = 0.0f;
    float FrameRateTimer = 0.0f;
 
    int32 ImageNumber = 0;
 
    uint8* BGR8Buffer = nullptr;
 
    int32 CurrentBufferSize = 0;
};