DiunaBI/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs

using System.Globalization;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context;
using DiunaBI.Core.Services;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;

namespace DiunaBI.Plugins.Morska.Importers;

public class MorskaD3Importer : MorskaBaseImporter
{
    public override string ImporterType => "Morska.Import.D3";

    private readonly AppDbContext _db;
    private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
    private readonly ILogger<MorskaD3Importer> _logger;

    // Configuration properties
    private string? ImportYear { get; set; }
    private string? ImportMonth { get; set; }
    private string? ImportName { get; set; }
    private string? ImportType { get; set; }
    private string? SheetId { get; set; }
    private bool IsEnabled { get; set; }

    // Cached deserialized data
    private List<Record>? _cachedRecords;
    private string? _cachedDataInboxId;

    public MorskaD3Importer(
        AppDbContext db,
        SpreadsheetsResource.ValuesResource googleSheetValues,
        ILogger<MorskaD3Importer> logger)
    {
        _db = db;
        _googleSheetValues = googleSheetValues;
        _logger = logger;
    }

    public override void Import(Layer importWorker)
    {
        try
        {
            _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
                ImporterType, importWorker.Name, importWorker.Id);

            // Clear cache at start
            _cachedRecords = null;
            _cachedDataInboxId = null;

            LoadConfiguration(importWorker);

            // Deserialize data early - right after LoadConfiguration
            DeserializeDataInboxData();

            if (!ShouldPerformImport(importWorker))
            {
                _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
                    ImporterType, importWorker.Name);
                return;
            }

            ValidateConfiguration();

            PerformImport(importWorker);

            // Export to Google Sheets after successful import
            ExportToGoogleSheets();

            _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
                ImporterType, importWorker.Name);
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
                ImporterType, importWorker.Name, importWorker.Id);
            throw;
        }
        finally
        {
            // Clear cache after import
            _cachedRecords = null;
            _cachedDataInboxId = null;
        }
    }

    private void LoadConfiguration(Layer importWorker)
    {
        if (importWorker.Records == null) return;

        ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
        ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
        ImportName = GetRecordValue(importWorker.Records, "ImportName");
        ImportType = GetRecordValue(importWorker.Records, "ImportType");
        SheetId = GetRecordValue(importWorker.Records, "SheetId");
        IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";

        _logger.LogDebug(
            "{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}, SheetId: {SheetId}",
            ImporterType, importWorker.Name, ImportType, SheetId);
    }

    private void DeserializeDataInboxData()
    {
        var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
        if (dataInbox == null)
        {
            throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
        }

        _logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
            ImporterType, dataInbox.Id, dataInbox.CreatedAt);

        try
        {
            var data = Convert.FromBase64String(dataInbox.Data);
            var jsonString = Encoding.UTF8.GetString(data);

            _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
                ImporterType, data.Length);

            var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
            if (records == null)
            {
                throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
            }

            _logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
                ImporterType, records.Count);

            // Cache the deserialized data
            _cachedRecords = records;
            _cachedDataInboxId = dataInbox.Id.ToString();
        }
        catch (FormatException e)
        {
            _logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxName}",
                ImporterType, ImportType);
            throw new InvalidOperationException($"Invalid base64 data in DataInbox: {ImportType}", e);
        }
        catch (JsonException e)
        {
            _logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxName}",
                ImporterType, ImportType);
            throw new InvalidOperationException($"Invalid JSON data in DataInbox: {ImportType}", e);
        }
    }

    private List<Record> GetFilteredRecords()
    {
        if (_cachedRecords == null)
        {
            throw new InvalidOperationException("Data has not been deserialized yet");
        }

        var filteredRecords = _cachedRecords.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
        if (filteredRecords.Count == 0)
        {
            throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
        }

        _logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
            ImporterType, filteredRecords.Count, ImportYear, ImportMonth);

        return filteredRecords;
    }

    private bool ShouldPerformImport(Layer importWorker)
    {
        if (!IsEnabled)
        {
            _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
                ImporterType, importWorker.Name);
            return false;
        }

        var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
        if (dataInbox == null)
        {
            _logger.LogDebug("{ImporterType}: No DataInbox found for type {ImportType}",
                ImporterType, ImportType);
            return false;
        }
        
        // Check if imported layer is up to date
        if (!IsImportedLayerUpToDate(importWorker))
        {
            _logger.LogDebug("{ImporterType}: Layer is out of date, import needed for {ImportWorkerName}",
                ImporterType, importWorker.Name);
            return true;
        }

        _logger.LogDebug("{ImporterType}: Layer is up to date for {ImportWorkerName}",
            ImporterType, importWorker.Name);
        return false;
    }

    private void ValidateConfiguration()
    {
        var errors = new List<string>();

        if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
        if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
        if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
        if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
        if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");

        if (errors.Any())
        {
            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
        }
    }

    private bool IsImportedLayerUpToDate(Layer importWorker)
    {
        var newestLayer = _db.Layers
            .Include(x => x.Records)
            .Where(x =>
                x.ParentId == importWorker.Id
                && x.Name!.Contains($"I-{ImportName}-{ImportYear}/{ImportMonth}"))
            .OrderByDescending(x => x.CreatedAt)
            .AsNoTracking()
            .FirstOrDefault();

        if (newestLayer == null)
        {
            _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, import needed",
                ImporterType, importWorker.Name);
            return false;
        }

        try
        {
            var currentRecords = GetFilteredRecords();

            // Compare record counts first
            if (newestLayer.Records?.Count != currentRecords.Count)
            {
                _logger.LogDebug("{ImporterType}: Record count mismatch - DB: {DbCount}, DataInbox: {DataCount}",
                    ImporterType, newestLayer.Records?.Count ?? 0, currentRecords.Count);
                return false;
            }

            // Compare individual records
            foreach (var currentRecord in currentRecords)
            {
                var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == currentRecord.Code);
                if (existingRecord == null)
                {
                    _logger.LogDebug("{ImporterType}: Record with code {Code} not found in existing layer",
                        ImporterType, currentRecord.Code);
                    return false;
                }

                // Compare all relevant fields
                if (Math.Abs((existingRecord.Value1 ?? 0) - (currentRecord.Value1 ?? 0)) > 0.001 ||
                    existingRecord.Desc1 != currentRecord.Desc1)
                {
                    _logger.LogDebug("{ImporterType}: Data difference found for code {Code}",
                        ImporterType, currentRecord.Code);
                    return false;
                }
            }

            _logger.LogDebug("{ImporterType}: All records match, layer is up to date for {ImportWorkerName}",
                ImporterType, importWorker.Name);
            return true;
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
                ImporterType, importWorker.Name);
            return false;
        }
    }

    private void PerformImport(Layer importWorker)
    {
        _logger.LogDebug("{ImporterType}: Starting import for DataInbox type {ImportType}",
            ImporterType, ImportType);

        var filteredRecords = GetFilteredRecords();

        // Prepare records for saving
        var recordsToSave = filteredRecords.Select(x =>
        {
            x.Id = Guid.NewGuid();
            x.CreatedAt = DateTime.UtcNow;
            x.ModifiedAt = DateTime.UtcNow;
            return x;
        }).ToList();

        var layer = new Layer
        {
            Id = Guid.NewGuid(),
            Number = _db.Layers.Count() + 1,
            ParentId = importWorker.Id,
            Type = LayerType.Import,
            CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
            ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
            CreatedAt = DateTime.UtcNow,
            ModifiedAt = DateTime.UtcNow
        };
        layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";

        _db.Layers.Add(layer);
        SaveRecords(layer.Id, recordsToSave);
        _db.SaveChanges();

        _logger.LogInformation(
            "{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
            ImporterType, recordsToSave.Count, layer.Name, layer.Id);
    }

    private void ExportToGoogleSheets()
    {
        if (string.IsNullOrEmpty(SheetId))
        {
            _logger.LogWarning("{ImporterType}: SheetId not configured, skipping Google Sheets export",
                ImporterType);
            return;
        }

        try
        {
            _logger.LogDebug("{ImporterType}: Starting Google Sheets export to {SheetId}",
                ImporterType, SheetId);

            var filteredRecords = GetFilteredRecords();
            var sheetTabName = ProcessHelper.GetSheetName(int.Parse(ImportMonth!), int.Parse(ImportYear!));

            _logger.LogDebug("{ImporterType}: Using sheet tab name: {SheetTabName}",
                ImporterType, sheetTabName);

            // Get current sheet data
            var currentSheetData = _googleSheetValues.Get(SheetId!, $"{sheetTabName}!C7:D200").Execute();

            var updateRequests = new List<ValueRange>();

            for (var rowIndex = 0; rowIndex < 194; rowIndex++)
            {
                if (currentSheetData.Values == null || rowIndex >= currentSheetData.Values.Count)
                    continue;

                var existingRow = currentSheetData.Values[rowIndex];
                if (existingRow.Count == 0 || string.IsNullOrEmpty(existingRow[0]?.ToString()))
                    continue;

                var accountCode = existingRow[0].ToString()!;
                var matchingRecord = filteredRecords.FirstOrDefault(x => x.Desc1 == accountCode);
                if (matchingRecord == null)
                    continue;

                var newValue = matchingRecord.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0";
                var existingValue = (existingRow.Count > 1 ? existingRow[1] : "")?.ToString();

                if (existingValue != newValue)
                {
                    var range = $"{sheetTabName}!D{rowIndex + 7}";
                    updateRequests.Add(new ValueRange
                    {
                        Range = range,
                        Values = new List<IList<object>> { new List<object> { newValue } }
                    });

                    _logger.LogDebug("{ImporterType}: Scheduled update for {AccountCode} at {Range} with value {Value}",
                        ImporterType, accountCode, range, newValue);
                }
            }

            if (updateRequests.Count > 0)
            {
                var batchUpdate = new BatchUpdateValuesRequest
                {
                    Data = updateRequests,
                    ValueInputOption = "USER_ENTERED"
                };

                _googleSheetValues.BatchUpdate(batchUpdate, SheetId!).Execute();

                _logger.LogInformation("{ImporterType}: Updated {Count} cells in Google Sheet {SheetId}",
                    ImporterType, updateRequests.Count, SheetId);
            }
            else
            {
                _logger.LogInformation("{ImporterType}: No changes to export to Google Sheet {SheetId}",
                    ImporterType, SheetId);
            }
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Failed to export to Google Sheets {SheetId}",
                ImporterType, SheetId);
            throw;
        }
    }

    private string? GetRecordValue(ICollection<Record> records, string code)
    {
        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
    }

    private void SaveRecords(Guid layerId, ICollection<Record> records)
    {
        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
        if (toDelete.Count > 0)
        {
            _db.Records.RemoveRange(toDelete);
        }

        foreach (var record in records)
        {
            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.CreatedAt = DateTime.UtcNow;
            record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.ModifiedAt = DateTime.UtcNow;
            record.LayerId = layerId;
            _db.Records.Add(record);
        }

        _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
            ImporterType, records.Count, layerId);
    }
}