ESMDataMapper.cpp

// Warning! Experimental! No guarantees on functionality! Use at your own risk!!!!!
#include "ESMDataMapper.h"
#include "Landscape.h"
#include "LandscapeInfo.h"
#include "LandscapeSubsystem.h"
#include "UObject/ConstructorHelpers.h"
#include "Misc/Guid.h"
#include "Logging/LogMacros.h"
#include "Engine/Texture2D.h"

// Extracts height data from a single LAND record in an ESM file.
// Parameters:
// - Record: The FRecord object containing LAND record data.
// - CellX, CellY: Expected cell coordinates to validate against the record.
// - OutHeightData: Output array to store height values (65x65 grid).
// - OutHeightOffset: Output float for the height offset from the VHGT subrecord.
// Returns: True if extraction succeeds, false otherwise.
bool UESMDataMapper::ExtractHeightDataFromRecord(const FRecord& Record, int32 CellX, int32 CellY, TArray<float>& OutHeightData, float& OutHeightOffset)
{
    // Log the record type and cell coordinates being processed.
    UE_LOG(LogTemp, Log, TEXT("Processing record with Type: '%s' for cell (%d, %d)"), *Record.Type, CellX, CellY);

    // Validate that the record is a LAND record.
    if (Record.Type != TEXT("LAND"))
    {
        UE_LOG(LogTemp, Warning, TEXT("Record is not a LAND record (Type: %s)"), *Record.Type);
        return false;
    }

    // Extract the INTV subrecord to validate cell coordinates.
    FSubrecord INTVSubrecord;
    if (URecordHelper::GetSubrecordByType(Record, TEXT("INTV"), INTVSubrecord) && INTVSubrecord.RawData.Num() >= sizeof(int32) * 2)
    {
        // Read the cell coordinates from the INTV subrecord.
        int32 RecordCellX = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData());
        int32 RecordCellY = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData() + sizeof(int32));
        // Ensure the record’s coordinates match the expected CellX, CellY.
        if (RecordCellX != CellX || RecordCellY != CellY)
        {
            UE_LOG(LogTemp, Warning, TEXT("Cell coordinates (%d, %d) do not match expected (%d, %d)"), RecordCellX, RecordCellY, CellX, CellY);
            return false;
        }
        UE_LOG(LogTemp, Log, TEXT("LAND record validated for cell (%d, %d)"), RecordCellX, RecordCellY);
    }
    else
    {
        UE_LOG(LogTemp, Error, TEXT("Invalid or missing INTV subrecord in LAND record"));
        return false;
    }

    // Extract the VHGT subrecord containing height data.
    FSubrecord VHGTSubrecord;
    if (!URecordHelper::GetSubrecordByType(Record, TEXT("VHGT"), VHGTSubrecord))
    {
        UE_LOG(LogTemp, Error, TEXT("Missing VHGT subrecord in LAND record"));
        OutHeightData.Empty();
        OutHeightOffset = 0.0f;
        return false;
    }

    // Validate the size of the VHGT subrecord data.
    // Expected size: sizeof(float) for HeightOffset + 1 byte padding + 65x65 height deltas + sizeof(uint16) padding.
    if (VHGTSubrecord.RawData.Num() < sizeof(float) + 1 + (65 * 65) + sizeof(uint16))
    {
        UE_LOG(LogTemp, Error, TEXT("VHGT subrecord data too small, expected >= 4232 bytes, got %d"), VHGTSubrecord.RawData.Num());
        OutHeightData.Empty();
        OutHeightOffset = 0.0f;
        return false;
    }

    // Read the height offset from the VHGT subrecord.
    OutHeightOffset = *reinterpret_cast<const float*>(VHGTSubrecord.RawData.GetData());
    UE_LOG(LogTemp, Log, TEXT("VHGT HeightOffset: %f"), OutHeightOffset);

    // Initialize the output height data array to a 65x65 grid.
    OutHeightData.SetNum(65 * 65);
    // Pointer to the height delta values in the VHGT subrecord (after the offset and padding).
    const int8* HeightDeltas = reinterpret_cast<const int8*>(VHGTSubrecord.RawData.GetData() + sizeof(float) + 1);
    // Start with the base height offset.
    float CurrentHeight = OutHeightOffset;

    // Iterate over the 65x65 grid to compute height values.
    for (int32 Y = 0; Y < 65; Y++)
    {
        for (int32 X = 0; X < 65; X++)
        {
            // Unreal Engine expects a flipped Y-axis, so adjust the index accordingly.
            int32 Index = (64 - Y) * 65 + X;
            int32 DataIndex = Y * 65 + X;
            // Accumulate the height delta to get the absolute height.
            CurrentHeight += HeightDeltas[DataIndex];
            // Scale the height by 8.0f (Morrowind to Unreal units conversion factor).
            OutHeightData[Index] = CurrentHeight * 8.0f;
        }
    }

    // Log the first 5 height values and the total size for debugging.
    for (int32 i = 0; i < FMath::Min(5, OutHeightData.Num()); i++)
    {
        UE_LOG(LogTemp, Log, TEXT("OutHeightData[%d]: %f"), i, OutHeightData[i]);
    }
    UE_LOG(LogTemp, Log, TEXT("OutHeightData size: %d"), OutHeightData.Num());

    return true;
}

// Extracts height data from an array of records for a specific cell.
// Parameters:
// - Records: Array of FRecord objects to search through.
// - CellX, CellY: Target cell coordinates to find a matching LAND record.
// - OutHeights: Output array to store height values.
// - OutHeightOffset: Output float for the height offset.
// Returns: True if a matching LAND record is found and extracted, false otherwise.
bool UESMDataMapper::ExtractHeightDataFromRecords(const TArray<FRecord>& Records, int32 CellX, int32 CellY, TArray<float>& OutHeights, float& OutHeightOffset)
{
    // Log the cell coordinates being processed.
    UE_LOG(LogTemp, Log, TEXT("ExtractHeightDataFromRecords called with CellX=%d, CellY=%d"), CellX, CellY);

    // Search for a LAND record matching the specified cell coordinates.
    FRecord LandRecord;
    bool bFound = false;
    for (const FRecord& Record : Records)
    {
        if (Record.Type == TEXT("LAND"))
        {
            FSubrecord INTVSubrecord;
            if (URecordHelper::GetSubrecordByType(Record, TEXT("INTV"), INTVSubrecord) && INTVSubrecord.RawData.Num() >= sizeof(int32) * 2)
            {
                int32 RecordCellX = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData());
                int32 RecordCellY = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData() + sizeof(int32));
                if (RecordCellX == CellX && RecordCellY == CellY)
                {
                    LandRecord = Record;
                    bFound = true;
                    break;
                }
            }
        }
    }

    // If no matching LAND record is found, log a warning and return false.
    if (!bFound)
    {
        UE_LOG(LogTemp, Warning, TEXT("No LAND record found for cell (%d, %d)"), CellX, CellY);
        return false;
    }

    // Extract height data from the found LAND record.
    return ExtractHeightDataFromRecord(LandRecord, CellX, CellY, OutHeights, OutHeightOffset);
}

// Extracts height data from LAND records in a plugin for a specific cell.
// Parameters:
// - Plugin: The FPlugin object containing ESM records.
// - CellX, CellY: Target cell coordinates.
// - OutHeights: Output array to store height values.
// - OutHeightOffset: Output float for the height offset.
// Returns: True if extraction succeeds, false otherwise.
bool UESMDataMapper::ExtractLandHeightDataFromPlugin(const FPlugin& Plugin, int32 CellX, int32 CellY, TArray<float>& OutHeights, float& OutHeightOffset)
{
    // Retrieve all records from the plugin.
    TArray<FRecord> Records = UPluginHelper::GetPluginRecords(Plugin);
    // Delegate to ExtractHeightDataFromRecords to process the records.
    return ExtractHeightDataFromRecords(Records, CellX, CellY, OutHeights, OutHeightOffset);
}

// Handles individual LAND record processing, extracting cell coordinates and height data.
// Parameters:
// - Record: The FRecord object to process.
// - CellX, CellY: Output parameters for the extracted cell coordinates.
// - OutHeightData: Output array for height values.
// - OutHeightOffset: Output float for the height offset.
// Returns: True if processing succeeds, false otherwise.
bool UESMDataMapper::LANDHandler(const FRecord& Record, int32& CellX, int32& CellY, TArray<float>& OutHeightData, float& OutHeightOffset)
{
    UE_LOG(LogTemp, Log, TEXT("Handling LAND record"));

    // Validate that the record is a LAND record.
    if (Record.Type != TEXT("LAND"))
    {
        UE_LOG(LogTemp, Warning, TEXT("Record is not a LAND record (Type: %s)"), *Record.Type);
        return false;
    }

    // Extract the INTV subrecord to get cell coordinates.
    FSubrecord INTVSubrecord;
    if (URecordHelper::GetSubrecordByType(Record, TEXT("INTV"), INTVSubrecord) && INTVSubrecord.RawData.Num() == 8)
    {
        // Log the raw data for debugging.
        UE_LOG(LogTemp, Log, TEXT("INTV subrecord raw data length: %d"), INTVSubrecord.RawData.Num());
        FString RawDataHex;
        for (uint8 Byte : INTVSubrecord.RawData)
        {
            RawDataHex += FString::Printf(TEXT("%02X "), Byte);
        }
        UE_LOG(LogTemp, Log, TEXT("INTV raw data (hex): %s"), *RawDataHex);

        // Extract CellX and CellY from the INTV subrecord.
        CellX = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData());
        CellY = *reinterpret_cast<const int32*>(INTVSubrecord.RawData.GetData() + sizeof(int32));
        UE_LOG(LogTemp, Log, TEXT("Extracted coordinates: CellX=%d, CellY=%d"), CellX, CellY);
    }
    else
    {
        UE_LOG(LogTemp, Error, TEXT("Invalid INTV subrecord: expected 8 bytes, got %d"), INTVSubrecord.RawData.Num());
        return false;
    }

    // Extract height data using the validated coordinates.
    return ExtractHeightDataFromRecord(Record, CellX, CellY, OutHeightData, OutHeightOffset);
}

// Converts raw height data (float) to a heightmap in byte format for Unreal Engine (legacy method).
// Parameters:
// - Heights: Input array of height values in float format.
// - MinHeight, MaxHeight: The height range for Unreal Engine (e.g., -2048 to 2048).
// Returns: A TArray<uint8> representing the heightmap as uint16 values.
TArray<uint8> UESMDataMapper::ConvertESMHeightToHeightmapBytes(const TArray<float>& Heights, float MinHeight, float MaxHeight)
{
    // Validate input data.
    if (Heights.Num() == 0)
    {
        UE_LOG(LogTemp, Warning, TEXT("No height data provided"));
        return TArray<uint8>();
    }

    // Initialize the heightmap array to store uint16 values.
    TArray<uint16> Heightmap;
    Heightmap.SetNum(Heights.Num());

    // Find the min and max height values in the input data.
    float DataMin = Heights[0];
    float DataMax = Heights[0];
    for (float Height : Heights)
    {
        DataMin = FMath::Min(DataMin, Height);
        DataMax = FMath::Max(DataMax, Height);
    }
    UE_LOG(LogTemp, Log, TEXT("Height data range: Min=%f, Max=%f"), DataMin, DataMax);

    // Normalize and scale the height values to the uint16 range (0-65535).
    float DataRange = DataMax - DataMin;
    float UnrealRange = MaxHeight - MinHeight;
    for (int32 i = 0; i < Heights.Num(); i++)
    {
        // Normalize the height to [0, 1] based on the data range.
        float Normalized = (DataRange > 0) ? (Heights[i] - DataMin) / DataRange : 0.5f;
        // Scale to Unreal’s height range.
        float ScaledHeight = MinHeight + (Normalized * UnrealRange);
        // Convert to uint16 (0-65535).
        Heightmap[i] = static_cast<uint16>((ScaledHeight - MinHeight) / UnrealRange * 65535);
        // Log the first 5 values for debugging.
        if (i < 5)
        {
            UE_LOG(LogTemp, Log, TEXT("Heightmap[%d]: %d (ScaledHeight=%f)"), i, Heightmap[i], ScaledHeight);
        }
    }

    // Convert the uint16 array to a byte array for Unreal Engine.
    TArray<uint8> Bytes;
    Bytes.SetNum(Heightmap.Num() * sizeof(uint16));
    FMemory::Memcpy(Bytes.GetData(), Heightmap.GetData(), Heightmap.Num() * sizeof(uint16));
    UE_LOG(LogTemp, Log, TEXT("HeightmapBytes size: %d"), Bytes.Num());
    return Bytes;
}

// Creates a landscape actor from a heightmap in byte format (legacy method, causes LandscapeGuid.IsValid() crash).
// Parameters:
// - World: The UWorld to spawn the landscape in.
// - HeightmapBytes: Byte array containing the heightmap as uint16 values.
// - Width, Height: Dimensions of the heightmap (e.g., 65x65).
// - ScaleX, ScaleY, ScaleZ: Scaling factors for the landscape.
// Returns: Pointer to the created ALandscape actor, or nullptr on failure.
ALandscape* UESMDataMapper::CreateLandscapeFromHeightmapBytes(
    UWorld* World,
    const TArray<uint8>& HeightmapBytes,
    int32 Width,
    int32 Height,
    float ScaleX,
    float ScaleY,
    float ScaleZ)
{
    // Validate the world pointer.
    if (!World)
    {
        UE_LOG(LogTemp, Error, TEXT("World is null"));
        return nullptr;
    }

    // Validate the heightmap size (should match Width * Height * sizeof(uint16)).
    if (HeightmapBytes.Num() == 0 || HeightmapBytes.Num() != Width * Height * sizeof(uint16))
    {
        UE_LOG(LogTemp, Error, TEXT("HeightmapBytes size (%d) does not match Width * Height * 2 (%d)"), HeightmapBytes.Num(), Width * Height * sizeof(uint16));
        return nullptr;
    }

    // Reinterpret the byte array as uint16 values.
    const uint16* HeightmapData = reinterpret_cast<const uint16*>(HeightmapBytes.GetData());

    // Spawn a new landscape actor.
    ALandscape* Landscape = World->SpawnActor<ALandscape>();
    if (!Landscape)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to spawn landscape actor"));
        return nullptr;
    }

    // Generate a new GUID for the landscape.
    FGuid LandscapeGuid = FGuid::NewGuid();
    if (!LandscapeGuid.IsValid())
    {
        UE_LOG(LogTemp, Error, TEXT("Generated LandscapeGuid is invalid"));
        Landscape->Destroy();
        return nullptr;
    }

    // Prepare the height data map for the Import function.
    TMap<FGuid, TArray<uint16>> HeightDataMap;
    TArray<uint16> HeightArray;
    HeightArray.SetNum(Width * Height);
    FMemory::Memcpy(HeightArray.GetData(), HeightmapData, Width * Height * sizeof(uint16));
    HeightDataMap.Add(LandscapeGuid, MoveTemp(HeightArray));

    // Create a LandscapeInfo object to manage metadata.
    ULandscapeInfo* LandscapeInfo = Landscape->CreateLandscapeInfo();
    if (!LandscapeInfo)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to create LandscapeInfo"));
        Landscape->Destroy();
        return nullptr;
    }

    // Assign the GUID to the LandscapeInfo.
    LandscapeInfo->LandscapeGuid = LandscapeGuid;

    // Import the heightmap into the landscape (this method causes the crash).
    Landscape->Import(
        LandscapeGuid,
        0,                      // Min X
        0,                      // Min Y
        Width - 1,              // Max X
        Height - 1,             // Max Y
        1,                      // Num subsections
        Width - 1,              // Subsection size quads
        HeightDataMap,          // Height data
        TEXT("None"),           // Material (none for now)
        TMap<FGuid, TArray<FLandscapeImportLayerInfo>>(), // No layers
        ELandscapeImportAlphamapType::Additive, // Alphamap type
        nullptr                 // No error message
    );

    // Apply the scaling to the landscape actor.
    Landscape->SetActorScale3D(FVector(ScaleX, ScaleY, ScaleZ));
    return Landscape;
}

// Creates a landscape actor using a modern Unreal Engine 5 approach with raw height data.
// Parameters:
// - World: The UWorld to spawn the landscape in.
// - Heights: Array of height values in float format.
// - CellX, CellY: Cell coordinates for positioning the landscape.
// - Width, Height: Dimensions of the heightmap (e.g., 65x65).
// - ScaleX, ScaleY, ScaleZ: Scaling factors for the landscape.
// Returns: Pointer to the created ALandscape actor, or nullptr on failure.
ALandscape* UESMDataMapper::CreateLandscapeModern(
    UWorld* World,
    const TArray<float>& Heights,
    int32 CellX,
    int32 CellY,
    int32 Width,
    int32 Height,
    float ScaleX,
    float ScaleY,
    float ScaleZ)
{
    // Validate the world pointer.
    if (!World)
    {
        UE_LOG(LogTemp, Error, TEXT("World is null"));
        return nullptr;
    }

    // Validate that the input heights array matches the expected size (Width * Height).
    if (Heights.Num() != Width * Height)
    {
        UE_LOG(LogTemp, Error, TEXT("Heights size (%d) does not match Width * Height (%d)"), Heights.Num(), Width * Height);
        return nullptr;
    }

    // Normalize height data to uint16 range (0-65535) for Unreal Engine.
    TArray<uint16> Heightmap;
    Heightmap.SetNum(Heights.Num());
    // Define the Unreal Engine height range.
    float MinHeight = -2048.0f;
    float MaxHeight = 2048.0f;
    // Find the min and max height values in the input data.
    float DataMin = Heights[0];
    float DataMax = Heights[0];
    for (float HeightValue : Heights)
    {
        DataMin = FMath::Min(DataMin, HeightValue);
        DataMax = FMath::Max(DataMax, HeightValue);
    }
    // Normalize and scale the heights.
    float DataRange = DataMax - DataMin;
    float UnrealRange = MaxHeight - MinHeight;
    for (int32 i = 0; i < Heights.Num(); i++)
    {
        // Normalize to [0, 1] based on the data range.
        float Normalized = (DataRange > 0) ? (Heights[i] - DataMin) / DataRange : 0.5f;
        // Scale to Unreal’s height range.
        float ScaledHeight = MinHeight + (Normalized * UnrealRange);
        // Convert to uint16 (0-65535).
        Heightmap[i] = static_cast<uint16>((ScaledHeight - MinHeight) / UnrealRange * 65535);
        // Log the first 5 values for debugging.
        if (i < 5)
        {
            UE_LOG(LogTemp, Log, TEXT("Heightmap[%d]: %d (ScaledHeight=%f)"), i, Heightmap[i], ScaledHeight);
        }
    }

    // Convert Heightmap (uint16) to FColor format for InitHeightmapData.
    TArray<FColor> HeightmapColors;
    HeightmapColors.SetNum(Heightmap.Num());
    for (int32 i = 0; i < Heightmap.Num(); i++)
    {
        // Landscape heightmaps store height in R (high byte) and G (low byte).
        HeightmapColors[i].R = (Heightmap[i] >> 8) & 0xFF; // High byte
        HeightmapColors[i].G = Heightmap[i] & 0xFF;        // Low byte
        HeightmapColors[i].B = 0;
        HeightmapColors[i].A = 0;
    }

    // Get the landscape subsystem to manage landscape creation.
    ULandscapeSubsystem* LandscapeSubsystem = World->GetSubsystem<ULandscapeSubsystem>();
    if (!LandscapeSubsystem)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to get LandscapeSubsystem"));
        return nullptr;
    }

    // Create or find a LandscapeInfo object to manage the landscape metadata.
    FGuid LandscapeGuid = FGuid::NewGuid();
    if (!LandscapeGuid.IsValid())
    {
        UE_LOG(LogTemp, Error, TEXT("Generated LandscapeGuid is invalid"));
        return nullptr;
    }

    ULandscapeInfo* LandscapeInfo = ULandscapeInfo::FindOrCreate(World, LandscapeGuid);
    if (!LandscapeInfo)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to create LandscapeInfo"));
        return nullptr;
    }

    // Spawn a new landscape actor.
    ALandscape* Landscape = World->SpawnActor<ALandscape>();
    if (!Landscape)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to spawn landscape actor"));
        return nullptr;
    }

    // Set basic landscape properties.
    Landscape->ComponentSizeQuads = Width - 1; // Quads = Verts - 1
    Landscape->SubsectionSizeQuads = Width - 1; // Single subsection for simplicity
    Landscape->SetActorScale3D(FVector(ScaleX, ScaleY, ScaleZ));

    // Calculate the section base position based on CellX, CellY.
    // Each cell is (Width-1) quads wide/tall, so SectionBase aligns the landscape in the world.
    FIntPoint SectionBase(CellX * (Width - 1), CellY * (Height - 1));

    // Find or add a landscape proxy using the subsystem.
    ALandscapeProxy* LandscapeProxy = LandscapeSubsystem->FindOrAddLandscapeProxy(LandscapeInfo, SectionBase);
    if (!LandscapeProxy)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to find or add LandscapeProxy"));
        Landscape->Destroy();
        return nullptr;
    }

    // Create a single landscape component to hold the heightmap data.
    ULandscapeComponent* Component = NewObject<ULandscapeComponent>(Landscape);
    if (!Component)
    {
        UE_LOG(LogTemp, Error, TEXT("Failed to create LandscapeComponent"));
        Landscape->Destroy();
        return nullptr;
    }

    // Set component properties to match the landscape dimensions.
    Component->ComponentSizeQuads = Width - 1;
    Component->SubsectionSizeQuads = Width - 1;
    Component->NumSubsections = 1;
    // Set the component's position using SetSectionBase.
    Component->SetSectionBase(SectionBase);

    // Initialize the component with the height data.
    Component->InitHeightmapData(HeightmapColors, false); // false for bUpdateCollision, as we’re not handling collision here

    // Register the component with the world and the LandscapeInfo.
    Component->RegisterComponent();
    LandscapeInfo->RegisterActorComponent(Component);

    // Update the LandscapeInfo with the landscape properties.
    LandscapeInfo->LandscapeActor = Landscape;
    LandscapeInfo->ComponentSizeQuads = Width - 1;
    LandscapeInfo->SubsectionSizeQuads = Width - 1;
    LandscapeInfo->ComponentNumSubsections = 1;
    LandscapeInfo->DrawScale = FVector(ScaleX, ScaleY, ScaleZ);

    // Log success message with the cell coordinates.
    UE_LOG(LogTemp, Log, TEXT("Successfully created landscape for cell (%d, %d) with height data applied"), CellX, CellY);
    return Landscape;
}