Morska.Import.Standard refactored

Utils/getProductionDatabase.sh

@@ -13,7 +13,7 @@ DOCKER_SA_PASSWORD="$&#ojoOOKEJ223"
 DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd"
 DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE}
 #SERVER VARIABLES
-REMOTE_HOST="crm.bim-it.pl"
+REMOTE_HOST="bim-it.pl"
 REMOTE_USER="mz"
 REMOTE_SA_PASSWORD="v](8Lc|RfG"
 REMOTE_BACKUP_DIR="/tmp"

Utils/totalCleanup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+docker stop $(docker ps -aq)
+docker rm -f $(docker ps -aq)
+docker rmi -f $(docker images -q)
+docker network prune -f
+docker volume prune -f
+docker builder prune -af

src/Backend/DiunaBI.Core/Models/Record.cs

@@ -45,7 +45,7 @@ public class Record
     public double? Value32 { get; set; }
     //Description fields
     [StringLength(10000)]
-    public string? Desc1 { get; init; }
+    public string? Desc1 { get; set; }
     public DateTime CreatedAt { get; set; }
     public DateTime ModifiedAt { get; set; }
     public bool IsDeleted { get; init; }

src/Backend/DiunaBI.Core/Services/ProcessHelper.cs

@@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
 using DiunaBI.Core.Models;
 
 namespace DiunaBI.Core.Services;
+
 public static class ProcessHelper
 {
     public static void SetValue(Record record, int number, double? value)
@@ -184,7 +185,6 @@ public static class ProcessHelper
         }
         return null;
     }
-
     public static string GetSheetName(int month, int year)
     {
         if (month < 1 || month > 12)

src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs

@@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers;
 
 public class MorskaD1Importer : MorskaBaseImporter
 {
-    public override string ImporterType => "MorskaD1";
+    public override string ImporterType => "Morska.Import.D1";
 
     private readonly AppDbContext _db;
     private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;

src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs

@@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Importers;
 
 public class MorskaD3Importer : MorskaBaseImporter
 {
-    public override string ImporterType => "MorskaD3";
+    public override string ImporterType => "Morska.Import.D3";
     
     private readonly AppDbContext _db;
     private readonly ILogger<MorskaD3Importer> _logger;

src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs

@@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers;
 
 public class MorskaFk2Importer : MorskaBaseImporter
 {
-    public override string ImporterType => "MorskaFK2";
+    public override string ImporterType => "Morska.Import.FK2";
     
     private readonly AppDbContext _db;
     private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;

src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs

@@ -1,6 +1,7 @@
 using System.Globalization;
 using Google.Apis.Sheets.v4;
 using Microsoft.Extensions.Logging;
+using Microsoft.EntityFrameworkCore;
 using DiunaBI.Core.Models;
 using DiunaBI.Database.Context;
 
@@ -8,12 +9,23 @@ namespace DiunaBI.Plugins.Morska.Importers;
 
 public class MorskaImporter : MorskaBaseImporter
 {
-    public override string ImporterType => "MorskaImporter";
+    public override string ImporterType => "Morska.Import.Standard";
 
     private readonly AppDbContext _db;
     private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
     private readonly ILogger<MorskaImporter> _logger;
 
+    // Configuration properties
+    private string? SheetId { get; set; }
+    private string? SheetTabName { get; set; }
+    private string? DataRange { get; set; }
+    private string? ImportYear { get; set; }
+    private string? ImportMonth { get; set; }
+    private string? ImportName { get; set; }
+    private DateTime? StartDate { get; set; }
+    private DateTime? EndDate { get; set; }
+    private bool IsEnabled { get; set; }
+
     public MorskaImporter(
         AppDbContext db,
         SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -26,47 +38,193 @@ public class MorskaImporter : MorskaBaseImporter
 
     public override void Import(Layer importWorker)
     {
-        _logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})", 
+        try
-            importWorker.Name, importWorker.Id);
-
-        var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1;
-        if (sheetId == null)
         {
-            throw new Exception($"SheetId not found, {importWorker.Name}");
+            _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
+                ImporterType, importWorker.Name, importWorker.Id);
+
+            // Load configuration from layer records
+            LoadConfiguration(importWorker);
+
+            // Check if import should be performed
+            if (!ShouldPerformImport(importWorker))
+            {
+                _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
+                    ImporterType, importWorker.Name);
+                return;
+            }
+
+            // Validate required configuration
+            ValidateConfiguration();
+
+            // Perform the actual import
+            PerformImport(importWorker);
+
+            _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
+                ImporterType, importWorker.Name);
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
+                ImporterType, importWorker.Name, importWorker.Id);
+            throw;
+        }
+    }
+
+    private void LoadConfiguration(Layer importWorker)
+    {
+        if (importWorker.Records == null) return;
+
+        SheetId = GetRecordValue(importWorker.Records, "SheetId");
+        SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
+        DataRange = GetRecordValue(importWorker.Records, "DataRange");
+        ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
+        ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
+        ImportName = GetRecordValue(importWorker.Records, "ImportName");
+        IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
+
+        var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
+        if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
+        {
+            StartDate = startDate;
         }
 
-        var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1;
+        var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
-        if (sheetTabName == null)
+        if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
         {
-            throw new Exception($"SheetTabName not found, {importWorker.Name}");
+            EndDate = endDate;
         }
 
-        var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1;
+        _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
-        if (year == null)
+            ImporterType, importWorker.Name);
+    }
+
+    private bool ShouldPerformImport(Layer importWorker)
+    {
+        if (!IsEnabled)
         {
-            throw new Exception($"ImportYear not found, {importWorker.Name}");
+            _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
+                ImporterType, importWorker.Name);
+            return false;
         }
 
-        var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1;
+        // Check date range if configured
-        if (month == null)
+        if (StartDate.HasValue && EndDate.HasValue)
         {
-            throw new Exception($"ImportMonth not found, {importWorker.Name}");
+            var now = DateTime.UtcNow.Date;
+            if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
+            {
+                _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
+                    ImporterType, importWorker.Name);
+                return true;
+            }
+
+            // Outside date range - check if imported layer is up to date
+            if (!IsImportedLayerUpToDate(importWorker))
+            {
+                _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
+                    ImporterType, importWorker.Name);
+                return true;
+            }
+
+            _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
+                ImporterType, importWorker.Name);
+            return false;
         }
 
-        var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1;
+        // No date constraints - always import
-        if (name == null)
+        return true;
+    }
+
+    private void ValidateConfiguration()
+    {
+        var errors = new List<string>();
+
+        if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
+        if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
+        if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
+        if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
+        if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
+        if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
+
+        if (errors.Any())
         {
-            throw new Exception($"ImportName not found, {importWorker.Name}");
+            throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
+        }
+    }
+
+    private bool IsImportedLayerUpToDate(Layer importWorker)
+    {
+        var newestLayer = _db.Layers
+            .Include(x => x.Records)
+            .Where(x => x.ParentId == importWorker.Id)
+            .OrderByDescending(x => x.CreatedAt)
+            .AsNoTracking()
+            .FirstOrDefault();
+
+        if (newestLayer == null)
+        {
+            _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
+                ImporterType, importWorker.Name);
+            return true;
         }
 
-        var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1;
+        try
-        if (dataRange == null)
         {
-            throw new Exception($"DataRange not found, {importWorker.Name}");
+            var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
+            var data = dataRangeResponse.Values;
+
+            if (data == null || data.Count < 2)
+            {
+                _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", 
+                    ImporterType, importWorker.Name);
+                return true; // Assume up to date if no data
+            }
+
+            // Check if the number of columns matches
+            if (data[0].Count != data[1].Count)
+            {
+                _logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
+                    ImporterType, importWorker.Name);
+                return false;
+            }
+
+            for (var i = 0; i < data[1].Count; i++)
+            {
+                if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
+                    !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
+                {
+                    _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", 
+                        ImporterType, i);
+                    continue;
+                }
+
+                // Check if the record exists in the database - add null check
+                var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
+                if (existingRecord == null || existingRecord.Value1 != value)
+                {
+                    _logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}", 
+                        ImporterType, data[0][i]);
+                    return false;
+                }
+            }
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
+                ImporterType, importWorker.Name);
+            return false;
         }
 
-        _logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
+        _logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
-            sheetId, sheetTabName, dataRange);
+            ImporterType, importWorker.Name);
+        return true;
+    }
+
+    private void PerformImport(Layer importWorker)
+    {
+        _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", 
+            ImporterType, SheetId, SheetTabName, DataRange);
 
         var layer = new Layer
         {
@@ -79,25 +237,29 @@ public class MorskaImporter : MorskaBaseImporter
             CreatedAt = DateTime.UtcNow,
             ModifiedAt = DateTime.UtcNow
         };
-        layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}";
+        layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
 
         var newRecords = new List<Record>();
 
         try
         {
-            var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
+            var dataRangeResponse = _googleSheetValues.Get(
+                SheetId!,
+                $"{SheetTabName}!{DataRange}").Execute();
             var data = dataRangeResponse.Values;
 
-            _logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0);
+            _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", 
+                ImporterType, data?.Count ?? 0);
 
             if (data != null && data.Count >= 2)
             {
                 for (var i = 0; i < data[1].Count; i++)
                 {
-                    if (!(data[0][i].ToString()?.Length > 0) ||
+                    if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
-                        !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
+                        !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
                     {
-                        _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i);
+                        _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", 
+                            ImporterType, i);
                         continue;
                     }
 
@@ -114,16 +276,16 @@ public class MorskaImporter : MorskaBaseImporter
             }
 
             _db.Layers.Add(layer);
-            
             SaveRecords(layer.Id, newRecords);
             _db.SaveChanges();
 
-            _logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
+            _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", 
-                newRecords.Count, layer.Name, layer.Id);
+                ImporterType, newRecords.Count, layer.Name, layer.Id);
         }
         catch (Exception e)
         {
-            _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId);
+            _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", 
+                ImporterType, SheetId);
             throw;
         }
     }
@@ -146,6 +308,12 @@ public class MorskaImporter : MorskaBaseImporter
             _db.Records.Add(record);
         }
 
-        _logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
+        _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
+            ImporterType, records.Count, layerId);
+    }
+
+    private string? GetRecordValue(ICollection<Record> records, string code)
+    {
+        return records.FirstOrDefault(x => x.Code == code)?.Desc1;
     }
 }

src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs

@@ -674,6 +674,112 @@ public class LayersController : Controller
         }
     }
 
+    [HttpGet]
+    [Route("GetImportWorkers")]
+    [AllowAnonymous]
+    public IActionResult GetImportWorkers()
+    {
+        var importWorkerLayers = _db.Layers
+            .Include(x => x.Records)
+            .Where(x =>
+                x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
+                x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
+                && x.Number == 3487
+            )
+            .OrderByDescending(x => x.CreatedAt)
+            .AsNoTracking()
+            .ToList();
+
+        foreach (var importWorker in importWorkerLayers)
+        {
+            _logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})",
+                importWorker.Name, importWorker.Id);
+            var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
+            if (pluginName != null)
+            {
+                var importer = _pluginManager.GetImporter(pluginName);
+                if (importer == null)
+                {
+                    _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})",
+                        pluginName, importWorker.Name, importWorker.Id);
+                    throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}");
+                }
+                try
+                {
+                    _logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}",
+                        importWorker.Name, importWorker.Id, pluginName);
+                    importer.Import(importWorker);
+                    _logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})",
+                        importWorker.Name, importWorker.Id);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}",
+                        importWorker.Name, importWorker.Id, pluginName);
+                    throw;
+                }
+            }
+            else
+            {
+                _logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})",
+                    importWorker.Name, importWorker.Id);
+                throw new Exception($"No plugin name found for import worker layer {importWorker.Name}");
+            }
+        }
+        return Ok();
+    }
+
+    [HttpGet]
+    [Route("AddPluginName")]
+    [AllowAnonymous]
+    public IActionResult AddPluginName()
+    {
+                    var record = new Record
+            {
+                Id = Guid.NewGuid(),
+                LayerId = Guid.Parse("eb5b4d0e-1607-4445-bbe5-65b9b8416787"),
+                Code = "Plugin",
+                Desc1 = "Morska.Import.Standard",
+                CreatedAt = DateTime.UtcNow,
+                ModifiedAt = DateTime.UtcNow,
+                CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
+            };
+            _db.Records.Add(record);
+        _db.SaveChanges();
+
+        /*
+        var importWorkerLayers = _db.Layers
+    .Include(x => x.Records)
+    .Where(x =>
+        x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
+        x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3")
+    )
+    .OrderByDescending(x => x.CreatedAt)
+    .AsNoTracking()
+    .ToList();
+
+        foreach (var importWorker in importWorkerLayers)
+        {
+            var record = new Record
+            {
+                Id = Guid.NewGuid(),
+                LayerId = importWorker.Id,
+                Code = "Plugin",
+                Desc1 = "Morska.Import.D3",
+                CreatedAt = DateTime.UtcNow,
+                ModifiedAt = DateTime.UtcNow,
+                CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
+                ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
+            };
+            //_db.Records.Add(record);
+        }
+
+        //_db.SaveChanges();
+        */
+        return Ok();
+    }
+
 
     private static void WriteToConsole(params string[] messages)
     {

src/Backend/DiunaBI.WebAPI/Program.cs

@@ -125,7 +125,9 @@ app.Use(async (context, next) =>
     if (token.Length > 0
         && !context.Request.Path.ToString().Contains("getForPowerBI")
         && !context.Request.Path.ToString().Contains("getConfiguration")
-        && !context.Request.Path.ToString().Contains("DataInbox/Add"))
+        && !context.Request.Path.ToString().Contains("DataInbox/Add")
+        && !context.Request.Path.ToString().Contains("AddPluginName") // TODO: Remove this 
+        && !context.Request.Path.ToString().Contains("GetImportWorkers"))
     {
         var handler = new JwtSecurityTokenHandler();
         var data = handler.ReadJwtToken(token.Split(' ')[1]);

src/Frontend/src/environments/environment.ts

@@ -7,8 +7,8 @@ export const environment = {
   appName: "LOCAL_DiunaBI",
   production: false,
   api: {
-    //url: "http://localhost:5400/api"
+    url: "http://localhost:5400/api"
-    url: "https://diunabi-morska.bim-it.pl/api"
+    //url: "https://diunabi-morska.bim-it.pl/api"
   },
   google: {
     clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com"

tools/http-tests/AutoImport.http

@@ -1,3 +1,3 @@
 ###
-GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5-
+GET http://localhost:5400/api/Layers/GetImportWorkers