DiunaBI/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs

using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context;
using DiunaBI.Core.Services;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4; 

namespace DiunaBI.Plugins.Morska.Processors;

public class MorskaT3SingleSourceProcessor : MorskaBaseProcessor
{
    public override string ProcessorType => "Morska.Process.T3.SingleSource";

    private readonly AppDbContext _db;
    private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
    private readonly ILogger<MorskaT3SingleSourceProcessor> _logger;

    // Configuration properties loaded from layer records
    private int Year { get; set; }
    private int Month { get; set; }
    private string? SourceLayerName { get; set; }
    private string? Source { get; set; }
    private Layer? SourceImportWorker { get; set; }

    public MorskaT3SingleSourceProcessor(
        AppDbContext db,
        SpreadsheetsResource.ValuesResource googleSheetValues,
        ILogger<MorskaT3SingleSourceProcessor> logger)
    {
        _db = db;
        _googleSheetValues = googleSheetValues;
        _logger = logger;
    }

    public override void Process(Layer processWorker)
    {
        try
        {
            _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
                ProcessorType, processWorker.Name, processWorker.Id);

            // Load configuration from layer records
            LoadConfiguration(processWorker);

            // Validate required configuration
            ValidateConfiguration();

            // Perform the actual processing
            PerformProcessing(processWorker);

            _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
                ProcessorType, processWorker.Name);
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
                ProcessorType, processWorker.Name, processWorker.Id);
            throw;
        }
    }

    private void LoadConfiguration(Layer processWorker)
    {
        if (processWorker.Records == null)
        {
            throw new InvalidOperationException("ProcessWorker has no records");
        }

        // Load year and month
        var yearStr = GetRecordValue(processWorker.Records, "Year");
        if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
        {
            throw new InvalidOperationException("Year record not found or invalid");
        }
        Year = year;

        var monthStr = GetRecordValue(processWorker.Records, "Month");
        if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
        {
            throw new InvalidOperationException("Month record not found or invalid");
        }
        Month = month;

        // Load source layer name
        SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer");
        if (string.IsNullOrEmpty(SourceLayerName))
        {
            throw new InvalidOperationException("SourceLayer record not found");
        }

        // Load source name
        Source = GetRecordValue(processWorker.Records, "Source");
        if (string.IsNullOrEmpty(Source))
        {
            throw new InvalidOperationException("Source record not found");
        }

        _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
            ProcessorType, Year, Month, SourceLayerName, Source);
    }

    private void ValidateConfiguration()
    {
        var errors = new List<string>();

        if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
        if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
        if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required");
        if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");

        // Find source import worker
        SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled);
        if (SourceImportWorker == null)
        {
            errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found");
        }

        if (errors.Any())
        {
            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
        }

        _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
    }

    private void PerformProcessing(Layer processWorker)
    {
        _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
            ProcessorType, Year, Month, Source);

        // Get or create processed layer
        var processedLayer = GetOrCreateProcessedLayer(processWorker);

        // Get data sources
        var dataSources = GetDataSources();

        // Process records
        var newRecords = ProcessRecords(dataSources);

        // Save results
        SaveProcessedLayer(processedLayer, newRecords);

        _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
            ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
    }

    private Layer GetOrCreateProcessedLayer(Layer processWorker)
    {
        var processedLayer = _db.Layers
            .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
            .OrderByDescending(x => x.CreatedAt)
            .FirstOrDefault();

        if (processedLayer == null)
        {
            processedLayer = new Layer
            {
                Id = Guid.NewGuid(),
                Type = LayerType.Processed,
                ParentId = processWorker.Id,
                Number = _db.Layers.Count() + 1,
                CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
                ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
                CreatedAt = DateTime.UtcNow,
                ModifiedAt = DateTime.UtcNow
            };
            processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3";

            _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", 
                ProcessorType, processedLayer.Name);
        }
        else
        {
            processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            processedLayer.ModifiedAt = DateTime.UtcNow;

            _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", 
                ProcessorType, processedLayer.Name);
        }

        return processedLayer;
    }

    private List<Layer> GetDataSources()
    {
        var dataSources = _db.Layers
            .Include(x => x.Records)
            .Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled)
            .OrderBy(x => x.CreatedAt)
            .AsNoTracking()
            .ToList();

        if (dataSources.Count == 0)
        {
            throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'");
        }

        _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
            ProcessorType, dataSources.Count);

        return dataSources;
    }

    private List<Record> ProcessRecords(List<Layer> dataSources)
    {
        var newRecords = new List<Record>();
        var allRecords = dataSources.SelectMany(x => x.Records!).ToList();

        _logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records",
            ProcessorType, allRecords.Count);

        foreach (var baseRecord in dataSources.Last().Records!)
        {
            var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();

            var processedRecord = new Record
            {
                Id = Guid.NewGuid(),
                Code = baseRecord.Code,
                CreatedAt = DateTime.UtcNow,
                ModifiedAt = DateTime.UtcNow
            };

            ProcessDailyValues(processedRecord, codeRecords);

            newRecords.Add(processedRecord);
        }

        _logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records",
            ProcessorType, newRecords.Count);

        return newRecords;
    }

    private void ProcessDailyValues(Record processedRecord, List<Record> codeRecords)
    {
        var lastDayInMonth = DateTime.DaysInMonth(Year, Month);

        // Day 1 - first value for the month
        var firstVal = codeRecords
            .Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1))
            .MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
        
        ProcessHelper.SetValue(processedRecord, 1, firstVal);
        var previousValue = firstVal;

        // Days 2 to last-1 - daily differences
        for (var i = 2; i < lastDayInMonth; i++)
        {
            var dayVal = codeRecords
                .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month)
                .MaxBy(x => x.CreatedAt)?.Value1;

            if (dayVal == null)
            {
                ProcessHelper.SetValue(processedRecord, i, 0);
            }
            else
            {
                var processedVal = dayVal - previousValue;
                ProcessHelper.SetValue(processedRecord, i, processedVal);
                previousValue = (double)dayVal;
            }
        }

        // Last day - special handling
        var lastVal = codeRecords
            .Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth))
            .MaxBy(x => x.CreatedAt)?.Value1;

        if (lastVal == null)
        {
            ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
        }
        else
        {
            ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
        }

        // Copy last value to position 32
        var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
        ProcessHelper.SetValue(processedRecord, 32, valueToCopy);

        _logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}",
            ProcessorType, processedRecord.Code, valueToCopy);
    }

    private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
    {
        var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id);

        if (isNew)
        {
            _db.Layers.Add(processedLayer);
            _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
        }
        else
        {
            _db.Layers.Update(processedLayer);
            _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
        }

        SaveRecords(processedLayer.Id, newRecords);
        _db.SaveChanges();

        _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
            ProcessorType, newRecords.Count, processedLayer.Id);
    }

    private void SaveRecords(Guid layerId, ICollection<Record> records)
    {
        // Remove existing records for this layer
        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
        if (toDelete.Count > 0)
        {
            _db.Records.RemoveRange(toDelete);
            _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
                ProcessorType, toDelete.Count, layerId);
        }

        // Add new records
        foreach (var record in records)
        {
            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.CreatedAt = DateTime.UtcNow;
            record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.ModifiedAt = DateTime.UtcNow;
            record.LayerId = layerId;
            _db.Records.Add(record);
        }

        _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
            ProcessorType, records.Count, layerId);
    }

    private string? GetRecordValue(ICollection<Record> records, string code)
    {
        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
    }
}