Morska.Import.D1 refactored

src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs

@@ -1,6 +1,7 @@
 using System.Globalization;
 using Google.Apis.Sheets.v4;
 using Microsoft.Extensions.Logging;
+using Microsoft.EntityFrameworkCore;
 using DiunaBI.Core.Models;
 using DiunaBI.Database.Context;
 
@@ -14,6 +15,17 @@ public class MorskaD1Importer : MorskaBaseImporter
     private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
     private readonly ILogger<MorskaD1Importer> _logger;
 
+    // Configuration properties
+    private string? SheetId { get; set; }
+    private string? SheetTabName { get; set; }
+    private string? DataRange { get; set; }
+    private string? ImportYear { get; set; }
+    private string? ImportMonth { get; set; }
+    private string? ImportName { get; set; }
+    private DateTime? StartDate { get; set; }
+    private DateTime? EndDate { get; set; }
+    private bool IsEnabled { get; set; }
+
     public MorskaD1Importer(
         AppDbContext db,
         SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -26,51 +38,195 @@ public class MorskaD1Importer : MorskaBaseImporter
 
     public override void Import(Layer importWorker)
     {
-        _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", 
+        try
-            importWorker.Name, importWorker.Id);
-
-        var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
-        if (sheetId == null)
         {
-            throw new Exception($"SheetId not found, {importWorker.Name}");
+            _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", 
+                ImporterType, importWorker.Name, importWorker.Id);
+
+            LoadConfiguration(importWorker);
+            
+            if (!ShouldPerformImport(importWorker))
+            {
+                _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", 
+                    ImporterType, importWorker.Name);
+                return;
+            }
+
+            ValidateConfiguration();
+
+            PerformImport(importWorker);
+            
+            _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", 
+                ImporterType, importWorker.Name);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", 
+                ImporterType, importWorker.Name, importWorker.Id);
+            throw;
+        }
+    }
+
+    private void LoadConfiguration(Layer importWorker)
+    {
+        if (importWorker.Records == null) return;
+
+        SheetId = GetRecordValue(importWorker.Records, "SheetId");
+        SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
+        DataRange = GetRecordValue(importWorker.Records, "DataRange");
+        ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
+        ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
+        ImportName = GetRecordValue(importWorker.Records, "ImportName");
+        IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
+
+        var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
+        if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
+        {
+            StartDate = startDate;
         }
 
-        var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
+        var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
-        if (sheetTabName == null)
+        if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
         {
-            throw new Exception($"SheetTabName not found, {importWorker.Name}");
+            EndDate = endDate;
         }
 
-        var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
+        _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", 
-        if (year == null)
+            ImporterType, importWorker.Name);
+    }
+
+    private bool ShouldPerformImport(Layer importWorker)
+    {
+        if (!IsEnabled)
         {
-            throw new Exception($"ImportYear not found, {importWorker.Name}");
+            _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", 
+                ImporterType, importWorker.Name);
+            return false;
         }
 
-        var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
+        if (StartDate.HasValue && EndDate.HasValue)
-        if (month == null)
         {
-            throw new Exception($"ImportMonth not found, {importWorker.Name}");
+            var now = DateTime.UtcNow.Date;
+            if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
+            {
+                _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", 
+                    ImporterType, importWorker.Name);
+                return true;
+            }
+            
+            if (!IsImportedLayerUpToDate(importWorker))
+            {
+                _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", 
+                    ImporterType, importWorker.Name);
+                return true;
+            }
+            
+            _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", 
+                ImporterType, importWorker.Name);
+            return false;
         }
 
-        var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
+        return true;
-        if (name == null)
+    }
+
+    private void ValidateConfiguration()
+    {
+        var errors = new List<string>();
+
+        if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
+        if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
+        if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
+        if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
+        if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
+        if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
+
+        if (errors.Any())
         {
-            throw new Exception($"ImportName not found, {importWorker.Name}");
+            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
+        }
+    }
+
+    private bool IsImportedLayerUpToDate(Layer importWorker)
+    {
+        var newestLayer = _db.Layers
+            .Include(x => x.Records)
+            .Where(x => x.ParentId == importWorker.Id)
+            .OrderByDescending(x => x.CreatedAt)
+            .AsNoTracking()
+            .FirstOrDefault();
+
+        if (newestLayer == null)
+        {
+            _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", 
+                ImporterType, importWorker.Name);
+            return true;
         }
 
-        var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
+        try
-        if (dataRange == null)
         {
-            throw new Exception($"DataRange not found, {importWorker.Name}");
+            var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
-        }
+            var data = dataRangeResponse.Values;
 
-        _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
+            if (data == null || data.Count < 2)
-            sheetId, sheetTabName, dataRange);
+            {
+                _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", 
+                    ImporterType, importWorker.Name);
+                return true;
+            }
+
+            var isUpToDate = true;
+
+            foreach (var row in data)
+            {
+                if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
+
+                var code = row[0].ToString();
+                var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
+                
+                if (record == null)
+                {
+                    _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", 
+                        ImporterType, code, importWorker.Name);
+                    isUpToDate = false;
+                    continue;
+                }
+
+                // Check values 3-17 (Value1-Value15)
+                for (int i = 3; i <= 17 && i < row.Count; i++)
+                {
+                    var sheetValue = ParseValue(row[i]?.ToString());
+                    var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
+
+                    if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
+                    {
+                        _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", 
+                            ImporterType, code, i - 2, dbValue, sheetValue);
+                        isUpToDate = false;
+                    }
+                }
+            }
+
+            _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", 
+                ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
+
+            return isUpToDate;
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", 
+                ImporterType, importWorker.Name);
+            throw;
+        }
+    }
+
+    private void PerformImport(Layer importWorker)
+    {
+        _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
+            ImporterType, SheetId, SheetTabName, DataRange);
 
         var layer = new Layer
         {
-            Id = Guid.NewGuid(), 
+            Id = Guid.NewGuid(),
             Number = _db.Layers.Count() + 1,
             ParentId = importWorker.Id,
             Type = LayerType.Import,
@@ -79,17 +235,18 @@ public class MorskaD1Importer : MorskaBaseImporter
             CreatedAt = DateTime.UtcNow,
             ModifiedAt = DateTime.UtcNow
         };
-        layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
+        layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
 
         try
         {
-            var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
+            var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
             var data = dataRangeResponse.Values;
 
-            _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
+            _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", 
+                ImporterType, data?.Count ?? 0);
 
             var newRecords = (from t in data
-                            where t.Count > 1 && (string)t[0] != string.Empty
+                            where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
                             select new Record
                             {
                                 Id = Guid.NewGuid(),
@@ -114,20 +271,70 @@ public class MorskaD1Importer : MorskaBaseImporter
                             }).ToList();
 
             _db.Layers.Add(layer);
-            
             SaveRecords(layer.Id, newRecords);
             _db.SaveChanges();
 
-            _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
+            _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
-                newRecords.Count, layer.Name, layer.Id);
+                ImporterType, newRecords.Count, layer.Name, layer.Id);
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId);
+            _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", 
+                ImporterType, SheetId);
             throw;
         }
     }
 
+    private double? ParseValue(string? value)
+    {
+        if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
+        value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
+        try
+        {
+            double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
+            return result;
+        }
+        catch (FormatException e)
+        {
+            _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", 
+                ImporterType, value, e.Message);
+            return null;
+        }
+    }
+
+    private bool IndexExists(IList<object> array, int index)
+    {
+        return array != null && index >= 0 && index < array.Count;
+    }
+
+    private double? GetRecordValueByIndex(Record record, int valueIndex)
+    {
+        return valueIndex switch
+        {
+            1 => record.Value1,
+            2 => record.Value2,
+            3 => record.Value3,
+            4 => record.Value4,
+            5 => record.Value5,
+            6 => record.Value6,
+            7 => record.Value7,
+            8 => record.Value8,
+            9 => record.Value9,
+            10 => record.Value10,
+            11 => record.Value11,
+            12 => record.Value12,
+            13 => record.Value13,
+            14 => record.Value14,
+            15 => record.Value15,
+            _ => null
+        };
+    }
+
+    private string? GetRecordValue(ICollection<Record> records, string code)
+    {
+        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
+    }
+
     private void SaveRecords(Guid layerId, ICollection<Record> records)
     {
         var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
@@ -146,27 +353,7 @@ public class MorskaD1Importer : MorskaBaseImporter
             _db.Records.Add(record);
         }
 
-        _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
+        _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", 
-    }
+            ImporterType, records.Count, layerId);
-
-    private double? ParseValue(string? value)
-    {
-        if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
-        value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
-        try
-        {
-            double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
-            return result;
-        }
-        catch (FormatException e)
-        {
-            _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message);
-            return null;
-        }
-    }
-
-    private bool IndexExists(IList<object> array, int index)
-    {
-        return array != null && index >= 0 && index < array.Count;
     }
 }

src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs

@@ -684,7 +684,7 @@ public class LayersController : Controller
             .Where(x =>
                 x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
                 x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
-                && x.Number == 3487
+                && x.Number == 8215
             )
             .OrderByDescending(x => x.CreatedAt)
             .AsNoTracking()