DiunaBI/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs

using System.Globalization;
using Google.Apis.Sheets.v4;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context;

namespace DiunaBI.Plugins.Morska.Importers;

public class MorskaD1Importer : MorskaBaseImporter
{
    public override string ImporterType => "Morska.Import.D1";

    private readonly AppDbContext _db;
    private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
    private readonly ILogger<MorskaD1Importer> _logger;

    // Configuration properties
    private string? SheetId { get; set; }
    private string? SheetTabName { get; set; }
    private string? DataRange { get; set; }
    private string? ImportYear { get; set; }
    private string? ImportMonth { get; set; }
    private string? ImportName { get; set; }
    private DateTime? StartDate { get; set; }
    private DateTime? EndDate { get; set; }
    private bool IsEnabled { get; set; }

    // Cache for sheet data
    private IList<IList<object>>? _cachedSheetData;
    private string? _cachedDataKey;

    public MorskaD1Importer(
        AppDbContext db,
        SpreadsheetsResource.ValuesResource googleSheetValues,
        ILogger<MorskaD1Importer> logger)
    {
        _db = db;
        _googleSheetValues = googleSheetValues;
        _logger = logger;
    }

    public override void Import(Layer importWorker)
    {
        try
        {
            _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", 
                ImporterType, importWorker.Name, importWorker.Id);
            
            _cachedSheetData = null;
            _cachedDataKey = null;

            LoadConfiguration(importWorker);
            
            if (!ShouldPerformImport(importWorker))
            {
                _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", 
                    ImporterType, importWorker.Name);
                return;
            }

            ValidateConfiguration();

            PerformImport(importWorker);
            
            _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", 
                ImporterType, importWorker.Name);
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", 
                ImporterType, importWorker.Name, importWorker.Id);
            throw;
        }
        finally
        {
            _cachedSheetData = null;
            _cachedDataKey = null;
        }
    }
    
    private IList<IList<object>>? GetSheetData()
    {
        var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
        
        if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
        {
            _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
                ImporterType, currentDataKey);
            return _cachedSheetData;
        }

        try
        {
            _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
                ImporterType, currentDataKey);

            var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
            _cachedSheetData = dataRangeResponse.Values;
            _cachedDataKey = currentDataKey;

            _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
                ImporterType, _cachedSheetData?.Count ?? 0);

            return _cachedSheetData;
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
                ImporterType, SheetId);
            throw;
        }
    }

    private void LoadConfiguration(Layer importWorker)
    {
        if (importWorker.Records == null) return;

        SheetId = GetRecordValue(importWorker.Records, "SheetId");
        SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
        DataRange = GetRecordValue(importWorker.Records, "DataRange");
        ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
        ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
        ImportName = GetRecordValue(importWorker.Records, "ImportName");
        IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";

        var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
        if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
        {
            StartDate = startDate;
        }

        var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
        if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
        {
            EndDate = endDate;
        }

        _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", 
            ImporterType, importWorker.Name);
    }

    private bool ShouldPerformImport(Layer importWorker)
    {
        if (!IsEnabled)
        {
            _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", 
                ImporterType, importWorker.Name);
            return false;
        }

        if (StartDate.HasValue && EndDate.HasValue)
        {
            var now = DateTime.UtcNow.Date;
            if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
            {
                _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", 
                    ImporterType, importWorker.Name);
                return true;
            }
            
            if (!IsImportedLayerUpToDate(importWorker))
            {
                _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", 
                    ImporterType, importWorker.Name);
                return true;
            }
            
            _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", 
                ImporterType, importWorker.Name);
            return false;
        }

        return true;
    }

    private void ValidateConfiguration()
    {
        var errors = new List<string>();

        if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
        if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
        if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
        if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
        if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
        if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");

        if (errors.Any())
        {
            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
        }
    }

    private bool IsImportedLayerUpToDate(Layer importWorker)
    {
        var newestLayer = _db.Layers
            .Include(x => x.Records)
            .Where(x => x.ParentId == importWorker.Id)
            .OrderByDescending(x => x.CreatedAt)
            .AsNoTracking()
            .FirstOrDefault();

        if (newestLayer == null)
        {
            _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", 
                ImporterType, importWorker.Name);
            return true;
        }

        try
        {
            var data = GetSheetData();

            if (data == null || data.Count < 2)
            {
                _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", 
                    ImporterType, importWorker.Name);
                return true;
            }

            var isUpToDate = true;

            foreach (var row in data)
            {
                if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;

                var code = row[0].ToString();
                var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
                
                if (record == null)
                {
                    _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", 
                        ImporterType, code, importWorker.Name);
                    isUpToDate = false;
                    continue;
                }

                // Check values 3-17 (Value1-Value15)
                for (int i = 3; i <= 17 && i < row.Count; i++)
                {
                    var sheetValue = ParseValue(row[i]?.ToString());
                    var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3

                    if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
                    {
                        _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", 
                            ImporterType, code, i - 2, dbValue, sheetValue);
                        isUpToDate = false;
                    }
                }
                // check also Desc1 from records (Value18 in GSheet)
                if (row.Count <= 18) continue;
                var sheetDesc1 = row[18]?.ToString();
                var recordDesc1 = record.Desc1;
                    
                var normalizedSheetDesc1 = string.IsNullOrEmpty(sheetDesc1) ? null : sheetDesc1;
                var normalizedRecordDesc1 = string.IsNullOrEmpty(recordDesc1) ? null : recordDesc1;

                if (normalizedSheetDesc1 == normalizedRecordDesc1) continue;
                _logger.LogDebug("{ImporterType}: Desc1 mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}", 
                    ImporterType, code, normalizedRecordDesc1, normalizedSheetDesc1);
                isUpToDate = false;
            }

            _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", 
                ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");

            return isUpToDate;
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", 
                ImporterType, importWorker.Name);
            throw;
        }
    }

    private void PerformImport(Layer importWorker)
    {
        _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
            ImporterType, SheetId, SheetTabName, DataRange);

        var layer = new Layer
        {
            Id = Guid.NewGuid(),
            Number = _db.Layers.Count() + 1,
            ParentId = importWorker.Id,
            Type = LayerType.Import,
            CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
            ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
            CreatedAt = DateTime.UtcNow,
            ModifiedAt = DateTime.UtcNow
        };
        layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";

        try
        {
            var data = GetSheetData();

            _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", 
                ImporterType, data?.Count ?? 0);

            var newRecords = (from t in data
                            where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
                            select new Record
                            {
                                Id = Guid.NewGuid(),
                                Code = t[0].ToString(),
                                Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
                                Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
                                Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
                                Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
                                Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
                                Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
                                Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
                                Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
                                Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
                                Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
                                Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
                                Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
                                Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
                                Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
                                Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
                                Desc1 = IndexExists(t, 18) ? t[18]?.ToString() : null,
                                CreatedAt = DateTime.UtcNow,
                                ModifiedAt = DateTime.UtcNow
                            }).ToList();

            _db.Layers.Add(layer);
            SaveRecords(layer.Id, newRecords);
            _db.SaveChanges();

            _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
                ImporterType, newRecords.Count, layer.Name, layer.Id);
        }
        catch (Exception e)
        {
            _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
            throw;
        }
    }

    private double? ParseValue(string? value)
    {
        if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
        value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
        try
        {
            double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
            return result;
        }
        catch (FormatException e)
        {
            _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", 
                ImporterType, value, e.Message);
            return null;
        }
    }

    private bool IndexExists(IList<object> array, int index)
    {
        return array != null && index >= 0 && index < array.Count;
    }

    private double? GetRecordValueByIndex(Record record, int valueIndex)
    {
        return valueIndex switch
        {
            1 => record.Value1,
            2 => record.Value2,
            3 => record.Value3,
            4 => record.Value4,
            5 => record.Value5,
            6 => record.Value6,
            7 => record.Value7,
            8 => record.Value8,
            9 => record.Value9,
            10 => record.Value10,
            11 => record.Value11,
            12 => record.Value12,
            13 => record.Value13,
            14 => record.Value14,
            15 => record.Value15,
            _ => null
        };
    }

    private string? GetRecordValue(ICollection<Record> records, string code)
    {
        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
    }

    private void SaveRecords(Guid layerId, ICollection<Record> records)
    {
        var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
        if (toDelete.Count > 0)
        {
            _db.Records.RemoveRange(toDelete);
        }

        foreach (var record in records)
        {
            record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.CreatedAt = DateTime.UtcNow;
            record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
            record.ModifiedAt = DateTime.UtcNow;
            record.LayerId = layerId;
            _db.Records.Add(record);
        }

        _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", 
            ImporterType, records.Count, layerId);
    }
}