From ee9307c7c31db6f00435a997c1e3ec71f6562d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 20:43:19 +0200 Subject: [PATCH] Morska.Import.D1 refactored --- .../Importers/MorskaD1Importer.cs | 297 ++++++++++++++---- .../Controllers/LayersController.cs | 2 +- 2 files changed, 243 insertions(+), 56 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index d509374..8e3b2af 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,17 @@ public class MorskaD1Importer : MorskaBaseImporter private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,51 +38,195 @@ public class MorskaD1Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + foreach (var row in data) + { + if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue; + + var code = row[0].ToString(); + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + // Check values 3-17 (Value1-Value15) + for (int i = 3; i <= 17 && i < row.Count; i++) + { + var sheetValue = ParseValue(row[i]?.ToString()); + var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3 + + if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, i - 2, dbValue, sheetValue); + isUpToDate = false; + } + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { - Id = Guid.NewGuid(), + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -79,17 +235,18 @@ public class MorskaD1Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); var newRecords = (from t in data - where t.Count > 1 && (string)t[0] != string.Empty + where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString()) select new Record { Id = Guid.NewGuid(), @@ -114,20 +271,70 @@ public class MorskaD1Importer : MorskaBaseImporter }).ToList(); _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } + private double? ParseValue(string? value) + { + if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; + value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); + try + { + double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); + return result; + } + catch (FormatException e) + { + _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", + ImporterType, value, e.Message); + return null; + } + } + + private bool IndexExists(IList array, int index) + { + return array != null && index >= 0 && index < array.Count; + } + + private double? GetRecordValueByIndex(Record record, int valueIndex) + { + return valueIndex switch + { + 1 => record.Value1, + 2 => record.Value2, + 3 => record.Value3, + 4 => record.Value4, + 5 => record.Value5, + 6 => record.Value6, + 7 => record.Value7, + 8 => record.Value8, + 9 => record.Value9, + 10 => record.Value10, + 11 => record.Value11, + 12 => record.Value12, + 13 => record.Value13, + 14 => record.Value14, + 15 => record.Value15, + _ => null + }; + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -146,27 +353,7 @@ public class MorskaD1Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private double? ParseValue(string? value) - { - if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; - value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); - try - { - double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); - return result; - } - catch (FormatException e) - { - _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message); - return null; - } - } - - private bool IndexExists(IList array, int index) - { - return array != null && index >= 0 && index < array.Count; + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index e26e973..6fa9b27 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -684,7 +684,7 @@ public class LayersController : Controller .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 3487 + && x.Number == 8215 ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking()