diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 0050947..f019bde 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.Json; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,15 @@ public class MorskaD3Importer : MorskaBaseImporter private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private string? ImportType { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaD3Importer( AppDbContext db, ILogger logger) @@ -24,67 +34,191 @@ public class MorskaD3Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + try { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + ImportType = GetRecordValue(importWorker.Records, "ImportType"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + EndDate = endDate; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}", + ImporterType, importWorker.Name, ImportType); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; - if (type == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportType not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - _logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type); + return true; + } - var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; + } + + try + { + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); + if (dataInbox == null) + { + _logger.LogWarning("{ImporterType}: No DataInbox found for type {ImportType}", + ImporterType, ImportType); + return true; // Assume up to date if no data source + } + + // Compare timestamps - if DataInbox is newer than our layer, we need to import + var isUpToDate = newestLayer.CreatedAt >= dataInbox.CreatedAt; + + _logger.LogDebug("{ImporterType}: Layer created at {LayerTime}, DataInbox created at {DataTime}, up to date: {IsUpToDate}", + ImporterType, newestLayer.CreatedAt, dataInbox.CreatedAt, isUpToDate); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Looking for DataInbox with type {ImportType}", + ImporterType, ImportType); + + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); if (dataInbox == null) { - throw new Exception($"DataInbox not found, {type}"); + throw new InvalidOperationException($"DataInbox not found for type: {ImportType}"); } - _logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}", - dataInbox.Id, dataInbox.CreatedAt); + _logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}", + ImporterType, dataInbox.Id, dataInbox.CreatedAt); try { var data = Convert.FromBase64String(dataInbox.Data); var jsonString = Encoding.UTF8.GetString(data); - _logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length); + _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64", + ImporterType, data.Length); var records = JsonSerializer.Deserialize>(jsonString); if (records == null) { - throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); + throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}"); } - _logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count); + _logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON", + ImporterType, records.Count); - records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); + records = records.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList(); if (records.Count == 0) { - throw new Exception($"No records found for {year}{month}"); + throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}"); } - _logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}", - records.Count, year, month); + _logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}", + ImporterType, records.Count, ImportYear, ImportMonth); records = records.Select(x => { @@ -105,23 +239,28 @@ public class MorskaD3Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; _db.Layers.Add(layer); - SaveRecords(layer.Id, records); _db.SaveChanges(); - _logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - records.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, records.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id); + _logger.LogError(e, "{ImporterType}: Error processing DataInbox {DataInboxId}", + ImporterType, dataInbox.Id); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -140,6 +279,7 @@ public class MorskaD3Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file