From edbb39c1cceb8334655df805d1359a6866a41c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 21:33:25 +0200 Subject: [PATCH] Morska.Import.FK2 refactored --- .../Importers/MorskaFK2Importer.cs | 253 +++++++++++++++--- .../Controllers/LayersController.cs | 2 +- 2 files changed, 218 insertions(+), 37 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 8b25067..f94ce47 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,17 @@ public class MorskaFk2Importer : MorskaBaseImporter private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaFk2Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,47 +38,206 @@ public class MorskaFk2Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + if (data == null || data.Count == 0) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + for (var i = 0; i < data.Count; i++) + { + if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue; + + try + { + var dateArr = data[i][1].ToString()!.Split("."); + if (dateArr.Length != 3) continue; + + var number = data[i][2].ToString()!; + if (number.Length == 1) number = $"0{number}"; + var code = dateArr[2] + dateArr[1] + dateArr[0] + number; + + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) && + Math.Abs((double)(record.Value1 - value)!) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, record.Value1, value); + isUpToDate = false; + } + + if (record.Desc1 != data[i][3]?.ToString()) + { + _logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}", + ImporterType, code, record.Desc1, data[i][3]?.ToString()); + isUpToDate = false; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison", + ImporterType, i); + isUpToDate = false; + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { @@ -79,16 +250,17 @@ public class MorskaFk2Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); if (data != null) { @@ -96,15 +268,17 @@ public class MorskaFk2Importer : MorskaBaseImporter { if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc", + ImporterType, i); continue; } var dateArr = data[i][1].ToString()!.Split("."); if (dateArr.Length != 3) { - _logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]); - throw new Exception($"Invalid date in row {i}"); + _logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}", + ImporterType, i, data[i][1]); + throw new InvalidOperationException($"Invalid date in row {i}"); } var number = data[i][2].ToString()!; @@ -114,7 +288,8 @@ public class MorskaFk2Importer : MorskaBaseImporter if (string.IsNullOrEmpty(data[i][9]?.ToString()) || !double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value", + ImporterType, i); continue; } @@ -132,20 +307,25 @@ public class MorskaFk2Importer : MorskaBaseImporter } _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -164,6 +344,7 @@ public class MorskaFk2Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 6fa9b27..faed569 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -684,7 +684,7 @@ public class LayersController : Controller .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 8215 + && x.Number == 7270 ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking()