using System.Globalization; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Plugins; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD1Importer : BaseDataImporter { public override string ImporterType => "Morska.Import.D1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties private string? SheetId { get; set; } private string? SheetTabName { get; set; } private string? DataRange { get; set; } private string? ImportYear { get; set; } private string? ImportMonth { get; set; } private string? ImportName { get; set; } private DateTime? StartDate { get; set; } private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } // Cache for sheet data private IList>? _cachedSheetData; private string? _cachedDataKey; public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Import(Layer importWorker) { try { _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); _cachedSheetData = null; _cachedDataKey = null; LoadConfiguration(importWorker); if (!ShouldPerformImport(importWorker)) { _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", ImporterType, importWorker.Name); return; } ValidateConfiguration(); PerformImport(importWorker); _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", ImporterType, importWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); throw; } finally { _cachedSheetData = null; _cachedDataKey = null; } } private IList>? GetSheetData() { var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; if (_cachedSheetData != null && _cachedDataKey == currentDataKey) { _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", ImporterType, currentDataKey); return _cachedSheetData; } try { _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", ImporterType, currentDataKey); var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); _cachedSheetData = dataRangeResponse.Values; _cachedDataKey = currentDataKey; _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", ImporterType, _cachedSheetData?.Count ?? 0); return _cachedSheetData; } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", ImporterType, SheetId); throw; } } private void LoadConfiguration(Layer importWorker) { if (importWorker.Records == null) return; SheetId = GetRecordValue(importWorker.Records, "SheetId"); SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); DataRange = GetRecordValue(importWorker.Records, "DataRange"); ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); ImportName = GetRecordValue(importWorker.Records, "ImportName"); IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) { StartDate = startDate; } var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { EndDate = endDate; } _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", ImporterType, importWorker.Name); } private bool ShouldPerformImport(Layer importWorker) { if (!IsEnabled) { _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", ImporterType, importWorker.Name); return false; } if (StartDate.HasValue && EndDate.HasValue) { var now = DateTime.UtcNow.Date; if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) { _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", ImporterType, importWorker.Name); return true; } if (!IsImportedLayerUpToDate(importWorker)) { _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", ImporterType, importWorker.Name); return true; } _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", ImporterType, importWorker.Name); return false; } return true; } private void ValidateConfiguration() { var errors = new List(); if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } } private bool IsImportedLayerUpToDate(Layer importWorker) { var newestLayer = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == importWorker.Id) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (newestLayer == null) { _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, need to create new layer", ImporterType, importWorker.Name); return false; } try { var data = GetSheetData(); if (data == null || data.Count < 2) { _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", ImporterType, importWorker.Name); return true; } var isUpToDate = true; foreach (var row in data) { if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue; var code = row[0].ToString(); var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); if (record == null) { _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", ImporterType, code, importWorker.Name); isUpToDate = false; continue; } // Check values 3-17 (Value1-Value15) for (int i = 3; i <= 17 && i < row.Count; i++) { var sheetValue = ParseValue(row[i]?.ToString()); var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3 if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01) { _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", ImporterType, code, i - 2, dbValue, sheetValue); isUpToDate = false; } } // check also Desc1 from records (Value18 in GSheet) if (row.Count <= 18) continue; var sheetDesc1 = row[18]?.ToString(); var recordDesc1 = record.Desc1; var normalizedSheetDesc1 = string.IsNullOrEmpty(sheetDesc1) ? null : sheetDesc1; var normalizedRecordDesc1 = string.IsNullOrEmpty(recordDesc1) ? null : recordDesc1; if (normalizedSheetDesc1 == normalizedRecordDesc1) continue; _logger.LogDebug("{ImporterType}: Desc1 mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}", ImporterType, code, normalizedRecordDesc1, normalizedSheetDesc1); isUpToDate = false; } _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", ImporterType, importWorker.Name); throw; } } private void PerformImport(Layer importWorker) { _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; try { var data = GetSheetData(); _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); var newRecords = (from t in data where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString()) select new Record { Id = Guid.NewGuid(), Code = t[0].ToString(), Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null, Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null, Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null, Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null, Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null, Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null, Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null, Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null, Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null, Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null, Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null, Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null, Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null, Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null, Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null, Desc1 = IndexExists(t, 18) ? t[18]?.ToString() : null, CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }).ToList(); _db.Layers.Add(layer); SaveRecords(layer.Id, newRecords); _db.SaveChanges(); _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } private double? ParseValue(string? value) { if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); try { double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); return result; } catch (FormatException e) { _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", ImporterType, value, e.Message); return null; } } private bool IndexExists(IList array, int index) { return array != null && index >= 0 && index < array.Count; } private double? GetRecordValueByIndex(Record record, int valueIndex) { return valueIndex switch { 1 => record.Value1, 2 => record.Value2, 3 => record.Value3, 4 => record.Value4, 5 => record.Value5, 6 => record.Value6, 7 => record.Value7, 8 => record.Value8, 9 => record.Value9, 10 => record.Value10, 11 => record.Value11, 12 => record.Value12, 13 => record.Value13, 14 => record.Value14, 15 => record.Value15, _ => null }; } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); } foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", ImporterType, records.Count, layerId); } }