using System.Globalization; using System.Text; using System.Text.Json; using DiunaBI.Domain.Entities; using DiunaBI.Infrastructure.Data; using DiunaBI.Infrastructure.Plugins; using DiunaBI.Infrastructure.Services; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD3Importer : BaseDataImporter { public override string ImporterType => "Morska.Import.D3"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; // Configuration properties private string? ImportYear { get; set; } private string? ImportMonth { get; set; } private string? ImportName { get; set; } private string? ImportType { get; set; } private string? SheetId { get; set; } private bool IsEnabled { get; set; } // Cached deserialized data private List? _cachedRecords; private string? _cachedDataInboxId; public MorskaD3Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; _logger = logger; } public override void Import(Layer importWorker) { try { _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); // Clear cache at start _cachedRecords = null; _cachedDataInboxId = null; LoadConfiguration(importWorker); // Deserialize data early - right after LoadConfiguration DeserializeDataInboxData(); if (!ShouldPerformImport(importWorker)) { _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", ImporterType, importWorker.Name); return; } ValidateConfiguration(); PerformImport(importWorker); // Export to Google Sheets after successful import ExportToGoogleSheets(); _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", ImporterType, importWorker.Name); } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); throw; } finally { // Clear cache after import _cachedRecords = null; _cachedDataInboxId = null; } } private void LoadConfiguration(Layer importWorker) { if (importWorker.Records == null) return; ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); ImportName = GetRecordValue(importWorker.Records, "ImportName"); ImportType = GetRecordValue(importWorker.Records, "ImportType"); SheetId = GetRecordValue(importWorker.Records, "SheetId"); IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; _logger.LogDebug( "{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}, SheetId: {SheetId}", ImporterType, importWorker.Name, ImportType, SheetId); } private void DeserializeDataInboxData() { var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); if (dataInbox == null) { throw new InvalidOperationException($"DataInbox not found for type: {ImportType}"); } _logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}", ImporterType, dataInbox.Id, dataInbox.CreatedAt); try { var data = Convert.FromBase64String(dataInbox.Data); var jsonString = Encoding.UTF8.GetString(data); _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64", ImporterType, data.Length); var records = JsonSerializer.Deserialize>(jsonString); if (records == null) { throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}"); } _logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON", ImporterType, records.Count); // Cache the deserialized data _cachedRecords = records; _cachedDataInboxId = dataInbox.Id.ToString(); } catch (FormatException e) { _logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxName}", ImporterType, ImportType); throw new InvalidOperationException($"Invalid base64 data in DataInbox: {ImportType}", e); } catch (JsonException e) { _logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxName}", ImporterType, ImportType); throw new InvalidOperationException($"Invalid JSON data in DataInbox: {ImportType}", e); } } private List GetFilteredRecords() { if (_cachedRecords == null) { throw new InvalidOperationException("Data has not been deserialized yet"); } var filteredRecords = _cachedRecords.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList(); if (filteredRecords.Count == 0) { throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}"); } _logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}", ImporterType, filteredRecords.Count, ImportYear, ImportMonth); return filteredRecords; } private bool ShouldPerformImport(Layer importWorker) { if (!IsEnabled) { _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", ImporterType, importWorker.Name); return false; } var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); if (dataInbox == null) { _logger.LogDebug("{ImporterType}: No DataInbox found for type {ImportType}", ImporterType, ImportType); return false; } // Check if imported layer is up to date if (!IsImportedLayerUpToDate(importWorker)) { _logger.LogDebug("{ImporterType}: Layer is out of date, import needed for {ImportWorkerName}", ImporterType, importWorker.Name); return true; } _logger.LogDebug("{ImporterType}: Layer is up to date for {ImportWorkerName}", ImporterType, importWorker.Name); return false; } private void ValidateConfiguration() { var errors = new List(); if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required"); if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); if (errors.Any()) { throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); } } private bool IsImportedLayerUpToDate(Layer importWorker) { var newestLayer = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == importWorker.Id && x.Name!.Contains($"I-{ImportName}-{ImportYear}/{ImportMonth}")) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (newestLayer == null) { _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, import needed", ImporterType, importWorker.Name); return false; } try { var currentRecords = GetFilteredRecords(); // Compare record counts first if (newestLayer.Records?.Count != currentRecords.Count) { _logger.LogDebug("{ImporterType}: Record count mismatch - DB: {DbCount}, DataInbox: {DataCount}", ImporterType, newestLayer.Records?.Count ?? 0, currentRecords.Count); return false; } // Compare individual records foreach (var currentRecord in currentRecords) { var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == currentRecord.Code); if (existingRecord == null) { _logger.LogDebug("{ImporterType}: Record with code {Code} not found in existing layer", ImporterType, currentRecord.Code); return false; } // Compare all relevant fields if (Math.Abs((existingRecord.Value1 ?? 0) - (currentRecord.Value1 ?? 0)) > 0.001 || existingRecord.Desc1 != currentRecord.Desc1) { _logger.LogDebug("{ImporterType}: Data difference found for code {Code}", ImporterType, currentRecord.Code); return false; } } _logger.LogDebug("{ImporterType}: All records match, layer is up to date for {ImportWorkerName}", ImporterType, importWorker.Name); return true; } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", ImporterType, importWorker.Name); return false; } } private void PerformImport(Layer importWorker) { _logger.LogDebug("{ImporterType}: Starting import for DataInbox type {ImportType}", ImporterType, ImportType); var filteredRecords = GetFilteredRecords(); // Prepare records for saving var recordsToSave = filteredRecords.Select(x => { x.Id = Guid.NewGuid(); x.CreatedAt = DateTime.UtcNow; x.ModifiedAt = DateTime.UtcNow; return x; }).ToList(); var layer = new Layer { Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; _db.Layers.Add(layer); SaveRecords(layer.Id, recordsToSave); _db.SaveChanges(); _logger.LogInformation( "{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", ImporterType, recordsToSave.Count, layer.Name, layer.Id); } private void ExportToGoogleSheets() { if (string.IsNullOrEmpty(SheetId)) { _logger.LogWarning("{ImporterType}: SheetId not configured, skipping Google Sheets export", ImporterType); return; } try { _logger.LogDebug("{ImporterType}: Starting Google Sheets export to {SheetId}", ImporterType, SheetId); var filteredRecords = GetFilteredRecords(); var sheetTabName = ProcessHelper.GetSheetName(int.Parse(ImportMonth!), int.Parse(ImportYear!)); _logger.LogDebug("{ImporterType}: Using sheet tab name: {SheetTabName}", ImporterType, sheetTabName); // Get current sheet data var currentSheetData = _googleSheetValues.Get(SheetId!, $"{sheetTabName}!C7:D200").Execute(); var updateRequests = new List(); for (var rowIndex = 0; rowIndex < 194; rowIndex++) { if (currentSheetData.Values == null || rowIndex >= currentSheetData.Values.Count) continue; var existingRow = currentSheetData.Values[rowIndex]; if (existingRow.Count == 0 || string.IsNullOrEmpty(existingRow[0]?.ToString())) continue; var accountCode = existingRow[0].ToString()!; var matchingRecord = filteredRecords.FirstOrDefault(x => x.Desc1 == accountCode); if (matchingRecord == null) continue; var newValue = matchingRecord.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0"; var existingValue = (existingRow.Count > 1 ? existingRow[1] : "")?.ToString(); if (existingValue != newValue) { var range = $"{sheetTabName}!D{rowIndex + 7}"; updateRequests.Add(new ValueRange { Range = range, Values = new List> { new List { newValue } } }); _logger.LogDebug("{ImporterType}: Scheduled update for {AccountCode} at {Range} with value {Value}", ImporterType, accountCode, range, newValue); } } if (updateRequests.Count > 0) { var batchUpdate = new BatchUpdateValuesRequest { Data = updateRequests, ValueInputOption = "USER_ENTERED" }; _googleSheetValues.BatchUpdate(batchUpdate, SheetId!).Execute(); _logger.LogInformation("{ImporterType}: Updated {Count} cells in Google Sheet {SheetId}", ImporterType, updateRequests.Count, SheetId); } else { _logger.LogInformation("{ImporterType}: No changes to export to Google Sheet {SheetId}", ImporterType, SheetId); } } catch (Exception e) { _logger.LogError(e, "{ImporterType}: Failed to export to Google Sheets {SheetId}", ImporterType, SheetId); throw; } } private string? GetRecordValue(ICollection records, string code) { return records.FirstOrDefault(x => x.Code == code)?.Desc1; } private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); } foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedAt = DateTime.UtcNow; record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.ModifiedAt = DateTime.UtcNow; record.LayerId = layerId; _db.Records.Add(record); } _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", ImporterType, records.Count, layerId); } }