Morska.Import.D3 refactored
This commit is contained in:
Michał Zieliński
@@ -2,6 +2,7 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using DiunaBI.Core.Models;
|
||||
using DiunaBI.Database.Context;
|
||||
|
||||
@@ -14,6 +15,15 @@ public class MorskaD3Importer : MorskaBaseImporter
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaD3Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private string? ImportType { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
public MorskaD3Importer(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaD3Importer> logger)
|
||||
@@ -24,67 +34,191 @@ public class MorskaD3Importer : MorskaBaseImporter
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
_logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
importWorker.Name, importWorker.Id);
|
||||
|
||||
var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
|
||||
if (year == null)
|
||||
try
|
||||
{
|
||||
throw new Exception($"ImportYear not found, {importWorker.Name}");
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
ImportType = GetRecordValue(importWorker.Records, "ImportType");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
|
||||
if (month == null)
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
throw new Exception($"ImportMonth not found, {importWorker.Name}");
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
|
||||
if (name == null)
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}",
|
||||
ImporterType, importWorker.Name, ImportType);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
throw new Exception($"ImportName not found, {importWorker.Name}");
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1;
|
||||
if (type == null)
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
throw new Exception($"ImportType not found, {importWorker.Name}");
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type);
|
||||
return true;
|
||||
}
|
||||
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type);
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No DataInbox found for type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
return true; // Assume up to date if no data source
|
||||
}
|
||||
|
||||
// Compare timestamps - if DataInbox is newer than our layer, we need to import
|
||||
var isUpToDate = newestLayer.CreatedAt >= dataInbox.CreatedAt;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer created at {LayerTime}, DataInbox created at {DataTime}, up to date: {IsUpToDate}",
|
||||
ImporterType, newestLayer.CreatedAt, dataInbox.CreatedAt, isUpToDate);
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Looking for DataInbox with type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
throw new Exception($"DataInbox not found, {type}");
|
||||
throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||
dataInbox.Id, dataInbox.CreatedAt);
|
||||
_logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||
ImporterType, dataInbox.Id, dataInbox.CreatedAt);
|
||||
|
||||
try
|
||||
{
|
||||
var data = Convert.FromBase64String(dataInbox.Data);
|
||||
var jsonString = Encoding.UTF8.GetString(data);
|
||||
|
||||
_logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length);
|
||||
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
|
||||
ImporterType, data.Length);
|
||||
|
||||
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
||||
if (records == null)
|
||||
{
|
||||
throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}");
|
||||
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count);
|
||||
_logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
|
||||
ImporterType, records.Count);
|
||||
|
||||
records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList();
|
||||
records = records.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
|
||||
if (records.Count == 0)
|
||||
{
|
||||
throw new Exception($"No records found for {year}{month}");
|
||||
throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||
records.Count, year, month);
|
||||
_logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||
ImporterType, records.Count, ImportYear, ImportMonth);
|
||||
|
||||
records = records.Select(x =>
|
||||
{
|
||||
@@ -105,23 +239,28 @@ public class MorskaD3Importer : MorskaBaseImporter
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
|
||||
SaveRecords(layer.Id, records);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
records.Count, layer.Name, layer.Id);
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, records.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id);
|
||||
_logger.LogError(e, "{ImporterType}: Error processing DataInbox {DataInboxId}",
|
||||
ImporterType, dataInbox.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
@@ -140,6 +279,7 @@ public class MorskaD3Importer : MorskaBaseImporter
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user