diff --git a/Utils/getProductionDatabase.sh b/Utils/getProductionDatabase.sh index a0be17f..6e229e1 100644 --- a/Utils/getProductionDatabase.sh +++ b/Utils/getProductionDatabase.sh @@ -13,7 +13,7 @@ DOCKER_SA_PASSWORD="$&#ojoOOKEJ223" DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd" DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE} #SERVER VARIABLES -REMOTE_HOST="crm.bim-it.pl" +REMOTE_HOST="bim-it.pl" REMOTE_USER="mz" REMOTE_SA_PASSWORD="v](8Lc|RfG" REMOTE_BACKUP_DIR="/tmp" diff --git a/Utils/totalCleanup.sh b/Utils/totalCleanup.sh new file mode 100644 index 0000000..f77ab94 --- /dev/null +++ b/Utils/totalCleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +docker stop $(docker ps -aq) +docker rm -f $(docker ps -aq) +docker rmi -f $(docker images -q) +docker network prune -f +docker volume prune -f +docker builder prune -af \ No newline at end of file diff --git a/src/Backend/DiunaBI.Database/Context/AppDbContext.cs b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs similarity index 64% rename from src/Backend/DiunaBI.Database/Context/AppDbContext.cs rename to src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs index 3b5b0e0..549d369 100644 --- a/src/Backend/DiunaBI.Database/Context/AppDbContext.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs @@ -2,7 +2,7 @@ using DiunaBI.Core.Models; using Microsoft.Extensions.Logging; -namespace DiunaBI.Database.Context; +namespace DiunaBI.Core.Database.Context; public class AppDbContext(DbContextOptions options) : DbContext(options) { @@ -21,14 +21,4 @@ public class AppDbContext(DbContextOptions options) : DbContext(op x.SourceId }); } - - private static readonly LoggerFactory MyLoggerFactory = - new(new[] { - new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider() - }); - - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - optionsBuilder.UseLoggerFactory(MyLoggerFactory); - } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs similarity index 85% rename from src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs rename to src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs index 12add8d..1da1ba1 100644 --- a/src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs @@ -4,24 +4,24 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Configuration; -namespace DiunaBI.Database.Context; +namespace DiunaBI.Core.Database.Context; public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory { public AppDbContext CreateDbContext(string[] args) { var configuration = new ConfigurationBuilder() - .SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI")) // ✅ Poprawna ścieżka + .SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI")) .AddJsonFile("appsettings.json", optional: false) .AddJsonFile("appsettings.Development.json", optional: true) .Build(); var optionsBuilder = new DbContextOptionsBuilder(); - var connectionString = configuration.GetConnectionString("DefaultConnection"); + var connectionString = configuration.GetConnectionString("SQLDatabase"); if (string.IsNullOrEmpty(connectionString)) { - throw new InvalidOperationException("Connection string 'DefaultConnection' not found in appsettings.json"); + throw new InvalidOperationException("Connection string 'SQLDatabase' not found in appsettings.json"); } optionsBuilder.UseSqlServer(connectionString); diff --git a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs similarity index 94% rename from src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs index e22e413..c0baa9b 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs index d1f6ba6..e5ac00e 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs index 3ca66f9..162b159 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs index e151324..1fdce1c 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs index a48de5a..07be6ad 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs index 54ead09..517ba43 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs index 5787223..ae1da37 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs index e07e1d1..c7afb66 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs index 96cc281..37af610 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs index fdc03a4..b19ab99 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs @@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs index b60890d..4b2411d 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs index c7faeaa..e226d0c 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs index fea2a0b..6ed6f16 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs index 5b047dd..5ab24ba 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs index 54160aa..803a714 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs index 6fc4823..a50b999 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs index 917717d..f2af26e 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.cs diff --git a/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs new file mode 100644 index 0000000..aec761a --- /dev/null +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs @@ -0,0 +1,415 @@ +// +using System; +using DiunaBI.Core.Database.Context; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DiunaBI.Core.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20250607084540_QueueJobRefactor")] + partial class QueueJobRefactor + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Data") + .IsRequired() + .HasMaxLength(2147483647) + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("DataInbox"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("IsCancelled") + .HasColumnType("bit"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Number") + .HasColumnType("int"); + + b.Property("ParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ModifiedById"); + + b.HasIndex("ParentId"); + + b.ToTable("Layers"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => + { + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("LayerId", "SourceId"); + + b.HasIndex("SourceId"); + + b.ToTable("ProcessSources"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CompletedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("JobType") + .HasColumnType("int"); + + b.Property("LastAttemptAt") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("LayerName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MaxRetries") + .HasColumnType("int"); + + b.Property("ModifiedAtUtc") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PluginName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Priority") + .HasColumnType("int"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.ToTable("QueueJobs"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Code") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Desc1") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Value1") + .HasColumnType("float"); + + b.Property("Value10") + .HasColumnType("float"); + + b.Property("Value11") + .HasColumnType("float"); + + b.Property("Value12") + .HasColumnType("float"); + + b.Property("Value13") + .HasColumnType("float"); + + b.Property("Value14") + .HasColumnType("float"); + + b.Property("Value15") + .HasColumnType("float"); + + b.Property("Value16") + .HasColumnType("float"); + + b.Property("Value17") + .HasColumnType("float"); + + b.Property("Value18") + .HasColumnType("float"); + + b.Property("Value19") + .HasColumnType("float"); + + b.Property("Value2") + .HasColumnType("float"); + + b.Property("Value20") + .HasColumnType("float"); + + b.Property("Value21") + .HasColumnType("float"); + + b.Property("Value22") + .HasColumnType("float"); + + b.Property("Value23") + .HasColumnType("float"); + + b.Property("Value24") + .HasColumnType("float"); + + b.Property("Value25") + .HasColumnType("float"); + + b.Property("Value26") + .HasColumnType("float"); + + b.Property("Value27") + .HasColumnType("float"); + + b.Property("Value28") + .HasColumnType("float"); + + b.Property("Value29") + .HasColumnType("float"); + + b.Property("Value3") + .HasColumnType("float"); + + b.Property("Value30") + .HasColumnType("float"); + + b.Property("Value31") + .HasColumnType("float"); + + b.Property("Value32") + .HasColumnType("float"); + + b.Property("Value4") + .HasColumnType("float"); + + b.Property("Value5") + .HasColumnType("float"); + + b.Property("Value6") + .HasColumnType("float"); + + b.Property("Value7") + .HasColumnType("float"); + + b.Property("Value8") + .HasColumnType("float"); + + b.Property("Value9") + .HasColumnType("float"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("LayerId"); + + b.HasIndex("ModifiedById"); + + b.ToTable("Records"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Email") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("UserName") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.Layer", "Parent") + .WithMany() + .HasForeignKey("ParentId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + + b.Navigation("Parent"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => + { + b.HasOne("DiunaBI.Core.Models.Layer", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => + { + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.Layer", null) + .WithMany("Records") + .HasForeignKey("LayerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.Navigation("Records"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs new file mode 100644 index 0000000..895d2f1 --- /dev/null +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs @@ -0,0 +1,170 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DiunaBI.Core.Migrations +{ + /// + public partial class QueueJobRefactor : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "Message", + table: "QueueJobs"); + + migrationBuilder.RenameColumn( + name: "Type", + table: "QueueJobs", + newName: "RetryCount"); + + migrationBuilder.RenameColumn( + name: "ModifiedAt", + table: "QueueJobs", + newName: "ModifiedAtUtc"); + + migrationBuilder.RenameColumn( + name: "Attempts", + table: "QueueJobs", + newName: "Priority"); + + migrationBuilder.AddColumn( + name: "CompletedAt", + table: "QueueJobs", + type: "datetime2", + nullable: true); + + migrationBuilder.AddColumn( + name: "CreatedAtUtc", + table: "QueueJobs", + type: "datetime2", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "CreatedById", + table: "QueueJobs", + type: "uniqueidentifier", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.AddColumn( + name: "JobType", + table: "QueueJobs", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "LastAttemptAt", + table: "QueueJobs", + type: "datetime2", + nullable: true); + + migrationBuilder.AddColumn( + name: "LastError", + table: "QueueJobs", + type: "nvarchar(1000)", + maxLength: 1000, + nullable: true); + + migrationBuilder.AddColumn( + name: "LayerName", + table: "QueueJobs", + type: "nvarchar(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "MaxRetries", + table: "QueueJobs", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "ModifiedById", + table: "QueueJobs", + type: "uniqueidentifier", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.AddColumn( + name: "PluginName", + table: "QueueJobs", + type: "nvarchar(100)", + maxLength: 100, + nullable: false, + defaultValue: ""); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CompletedAt", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "CreatedAtUtc", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "CreatedById", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "JobType", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LastAttemptAt", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LastError", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LayerName", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "MaxRetries", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "ModifiedById", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "PluginName", + table: "QueueJobs"); + + migrationBuilder.RenameColumn( + name: "RetryCount", + table: "QueueJobs", + newName: "Type"); + + migrationBuilder.RenameColumn( + name: "Priority", + table: "QueueJobs", + newName: "Attempts"); + + migrationBuilder.RenameColumn( + name: "ModifiedAtUtc", + table: "QueueJobs", + newName: "ModifiedAt"); + + migrationBuilder.AddColumn( + name: "Message", + table: "QueueJobs", + type: "nvarchar(max)", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs similarity index 81% rename from src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs index 0907c7e..c325531 100644 --- a/src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs @@ -1,11 +1,10 @@ // using System; +using DiunaBI.Core.Database.Context; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; #nullable disable @@ -18,12 +17,12 @@ namespace DiunaBI.Core.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "9.0.0") + .HasAnnotation("ProductVersion", "8.0.0") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - modelBuilder.Entity("WebAPI.Models.DataInbox", b => + modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -52,7 +51,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("DataInbox"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -101,7 +100,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("Layers"); }); - modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => { b.Property("LayerId") .HasColumnType("uniqueidentifier"); @@ -116,32 +115,63 @@ namespace DiunaBI.Core.Migrations b.ToTable("ProcessSources"); }); - modelBuilder.Entity("WebAPI.Models.QueueJob", b => + modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("uniqueidentifier"); - b.Property("Attempts") - .HasColumnType("int"); + b.Property("CompletedAt") + .HasColumnType("datetime2"); b.Property("CreatedAt") .HasColumnType("datetime2"); + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("JobType") + .HasColumnType("int"); + + b.Property("LastAttemptAt") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + b.Property("LayerId") .HasColumnType("uniqueidentifier"); - b.Property("Message") + b.Property("LayerName") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); - b.Property("ModifiedAt") - .HasColumnType("datetime2"); - - b.Property("Status") + b.Property("MaxRetries") .HasColumnType("int"); - b.Property("Type") + b.Property("ModifiedAtUtc") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PluginName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Priority") + .HasColumnType("int"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("Status") .HasColumnType("int"); b.HasKey("Id"); @@ -149,7 +179,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("QueueJobs"); }); - modelBuilder.Entity("WebAPI.Models.Record", b => + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -289,7 +319,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("Records"); }); - modelBuilder.Entity("WebAPI.Models.User", b => + modelBuilder.Entity("DiunaBI.Core.Models.User", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -311,21 +341,21 @@ namespace DiunaBI.Core.Migrations b.ToTable("Users"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { - b.HasOne("WebAPI.Models.User", "CreatedBy") + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") .WithMany() .HasForeignKey("CreatedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.User", "ModifiedBy") + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") .WithMany() .HasForeignKey("ModifiedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.Layer", "Parent") + b.HasOne("DiunaBI.Core.Models.Layer", "Parent") .WithMany() .HasForeignKey("ParentId"); @@ -336,9 +366,9 @@ namespace DiunaBI.Core.Migrations b.Navigation("Parent"); }); - modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => { - b.HasOne("WebAPI.Models.Layer", "Source") + b.HasOne("DiunaBI.Core.Models.Layer", "Source") .WithMany() .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -347,21 +377,21 @@ namespace DiunaBI.Core.Migrations b.Navigation("Source"); }); - modelBuilder.Entity("WebAPI.Models.Record", b => + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => { - b.HasOne("WebAPI.Models.User", "CreatedBy") + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") .WithMany() .HasForeignKey("CreatedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.Layer", null) + b.HasOne("DiunaBI.Core.Models.Layer", null) .WithMany("Records") .HasForeignKey("LayerId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.User", "ModifiedBy") + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") .WithMany() .HasForeignKey("ModifiedById") .OnDelete(DeleteBehavior.Cascade) @@ -372,7 +402,7 @@ namespace DiunaBI.Core.Migrations b.Navigation("ModifiedBy"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { b.Navigation("Records"); }); diff --git a/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj b/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj index 1920750..0a2988e 100644 --- a/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj +++ b/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj @@ -9,7 +9,21 @@ - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Models/QueueJob.cs b/src/Backend/DiunaBI.Core/Models/QueueJob.cs index 54f5fdb..e7a4a2c 100644 --- a/src/Backend/DiunaBI.Core/Models/QueueJob.cs +++ b/src/Backend/DiunaBI.Core/Models/QueueJob.cs @@ -3,27 +3,68 @@ using System.ComponentModel.DataAnnotations; namespace DiunaBI.Core.Models; -public enum JobStatus +public class QueueJob { - New, - Failed, - Success + [Key] + public Guid Id { get; set; } = Guid.NewGuid(); + + [Required] + public Guid LayerId { get; set; } + + [Required] + [MaxLength(200)] + public string LayerName { get; set; } = string.Empty; + + [Required] + [MaxLength(100)] + public string PluginName { get; set; } = string.Empty; + + [Required] + public JobType JobType { get; set; } + + public int Priority { get; set; } = 0; // 0 = highest priority + + [Required] + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + public int RetryCount { get; set; } = 0; + + public int MaxRetries { get; set; } = 5; + + [Required] + public JobStatus Status { get; set; } = JobStatus.Pending; + + [MaxLength(1000)] + public string? LastError { get; set; } + + public DateTime? LastAttemptAt { get; set; } + + public DateTime? CompletedAt { get; set; } + + [Required] + public Guid CreatedById { get; set; } + + [Required] + public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow; + + [Required] + public Guid ModifiedById { get; set; } + + [Required] + public DateTime ModifiedAtUtc { get; set; } = DateTime.UtcNow; } public enum JobType { - ImportWorker, - ProcessWorker + Import = 0, + Process = 1 } -public class QueueJob +public enum JobStatus { - [Key] public Guid Id { get; set; } - [Required] public Guid LayerId { get; set; } - [Required] public int Attempts { get; set; } - [Required] public JobStatus Status { get; set; } = JobStatus.New; - [Required] public JobType Type { get; set; } = JobType.ImportWorker; - public string Message { get; set; } = string.Empty; - [Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; - [Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow; + Pending, + Running, + Completed, + Failed, + Retrying } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Models/Record.cs b/src/Backend/DiunaBI.Core/Models/Record.cs index f791f06..a554c56 100644 --- a/src/Backend/DiunaBI.Core/Models/Record.cs +++ b/src/Backend/DiunaBI.Core/Models/Record.cs @@ -45,7 +45,7 @@ public class Record public double? Value32 { get; set; } //Description fields [StringLength(10000)] - public string? Desc1 { get; init; } + public string? Desc1 { get; set; } public DateTime CreatedAt { get; set; } public DateTime ModifiedAt { get; set; } public bool IsDeleted { get; init; } diff --git a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs index 46c7b0c..dcb3c8d 100644 --- a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs +++ b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs @@ -4,6 +4,7 @@ using System.Text.RegularExpressions; using DiunaBI.Core.Models; namespace DiunaBI.Core.Services; + public static class ProcessHelper { public static void SetValue(Record record, int number, double? value) @@ -184,7 +185,6 @@ public static class ProcessHelper } return null; } - public static string GetSheetName(int month, int year) { if (month < 1 || month > 12) diff --git a/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj b/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj deleted file mode 100644 index e3d6e0b..0000000 --- a/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net8.0 - enable - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj index f208118..a3345a5 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj +++ b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj @@ -12,6 +12,5 @@ - \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 322cba4..aec4c02 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -1,19 +1,35 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD1Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD1"; + public override string ImporterType => "Morska.Import.D1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,51 +42,239 @@ public class MorskaD1Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + try { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + EndDate = endDate; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"DataRange not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + try + { + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); + + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + foreach (var row in data) + { + if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue; + + var code = row[0].ToString(); + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + // Check values 3-17 (Value1-Value15) + for (int i = 3; i <= 17 && i < row.Count; i++) + { + var sheetValue = ParseValue(row[i]?.ToString()); + var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3 + + if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, i - 2, dbValue, sheetValue); + isUpToDate = false; + } + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { - Id = Guid.NewGuid(), + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -79,17 +283,18 @@ public class MorskaD1Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", + ImporterType, data?.Count ?? 0); var newRecords = (from t in data - where t.Count > 1 && (string)t[0] != string.Empty + where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString()) select new Record { Id = Guid.NewGuid(), @@ -114,20 +319,69 @@ public class MorskaD1Importer : MorskaBaseImporter }).ToList(); _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } + private double? ParseValue(string? value) + { + if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; + value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); + try + { + double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); + return result; + } + catch (FormatException e) + { + _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", + ImporterType, value, e.Message); + return null; + } + } + + private bool IndexExists(IList array, int index) + { + return array != null && index >= 0 && index < array.Count; + } + + private double? GetRecordValueByIndex(Record record, int valueIndex) + { + return valueIndex switch + { + 1 => record.Value1, + 2 => record.Value2, + 3 => record.Value3, + 4 => record.Value4, + 5 => record.Value5, + 6 => record.Value6, + 7 => record.Value7, + 8 => record.Value8, + 9 => record.Value9, + 10 => record.Value10, + 11 => record.Value11, + 12 => record.Value12, + 13 => record.Value13, + 14 => record.Value14, + 15 => record.Value15, + _ => null + }; + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -146,27 +400,7 @@ public class MorskaD1Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private double? ParseValue(string? value) - { - if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; - value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); - try - { - double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); - return result; - } - catch (FormatException e) - { - _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message); - return null; - } - } - - private bool IndexExists(IList array, int index) - { - return array != null && index >= 0 && index < array.Count; + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 59f5afb..ffae136 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -2,18 +2,28 @@ using System.Text; using System.Text.Json; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD3Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD3"; + public override string ImporterType => "Morska.Import.D3"; private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private string? ImportType { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaD3Importer( AppDbContext db, ILogger logger) @@ -24,67 +34,191 @@ public class MorskaD3Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + try { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + ImportType = GetRecordValue(importWorker.Records, "ImportType"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + EndDate = endDate; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}", + ImporterType, importWorker.Name, ImportType); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; - if (type == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportType not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - _logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type); + return true; + } - var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; + } + + try + { + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); + if (dataInbox == null) + { + _logger.LogWarning("{ImporterType}: No DataInbox found for type {ImportType}", + ImporterType, ImportType); + return true; // Assume up to date if no data source + } + + // Compare timestamps - if DataInbox is newer than our layer, we need to import + var isUpToDate = newestLayer.CreatedAt >= dataInbox.CreatedAt; + + _logger.LogDebug("{ImporterType}: Layer created at {LayerTime}, DataInbox created at {DataTime}, up to date: {IsUpToDate}", + ImporterType, newestLayer.CreatedAt, dataInbox.CreatedAt, isUpToDate); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Looking for DataInbox with type {ImportType}", + ImporterType, ImportType); + + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); if (dataInbox == null) { - throw new Exception($"DataInbox not found, {type}"); + throw new InvalidOperationException($"DataInbox not found for type: {ImportType}"); } - _logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}", - dataInbox.Id, dataInbox.CreatedAt); + _logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}", + ImporterType, dataInbox.Id, dataInbox.CreatedAt); try { var data = Convert.FromBase64String(dataInbox.Data); var jsonString = Encoding.UTF8.GetString(data); - _logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length); + _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64", + ImporterType, data.Length); var records = JsonSerializer.Deserialize>(jsonString); if (records == null) { - throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); + throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}"); } - _logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count); + _logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON", + ImporterType, records.Count); - records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); + records = records.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList(); if (records.Count == 0) { - throw new Exception($"No records found for {year}{month}"); + throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}"); } - _logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}", - records.Count, year, month); + _logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}", + ImporterType, records.Count, ImportYear, ImportMonth); records = records.Select(x => { @@ -105,23 +239,28 @@ public class MorskaD3Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; _db.Layers.Add(layer); - SaveRecords(layer.Id, records); _db.SaveChanges(); - _logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - records.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, records.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id); + _logger.LogError(e, "{ImporterType}: Error processing DataInbox {DataInboxId}", + ImporterType, dataInbox.Id); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -140,6 +279,7 @@ public class MorskaD3Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 62c9de0..e4ccda7 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -1,19 +1,35 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; public class MorskaFk2Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaFK2"; + public override string ImporterType => "Morska.Import.FK2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaFk2Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,47 +42,250 @@ public class MorskaFk2Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + try { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + EndDate = endDate; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"DataRange not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - _logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + try + { + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); + + if (data == null || data.Count == 0) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + for (var i = 0; i < data.Count; i++) + { + if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue; + + try + { + var dateArr = data[i][1].ToString()!.Split("."); + if (dateArr.Length != 3) continue; + + var number = data[i][2].ToString()!; + if (number.Length == 1) number = $"0{number}"; + var code = dateArr[2] + dateArr[1] + dateArr[0] + number; + + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) && + Math.Abs((double)(record.Value1 - value)!) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, record.Value1, value); + isUpToDate = false; + } + + if (record.Desc1 != data[i][3]?.ToString()) + { + _logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}", + ImporterType, code, record.Desc1, data[i][3]?.ToString()); + isUpToDate = false; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison", + ImporterType, i); + isUpToDate = false; + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { @@ -79,16 +298,17 @@ public class MorskaFk2Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", + ImporterType, data?.Count ?? 0); if (data != null) { @@ -96,15 +316,17 @@ public class MorskaFk2Importer : MorskaBaseImporter { if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc", + ImporterType, i); continue; } var dateArr = data[i][1].ToString()!.Split("."); if (dateArr.Length != 3) { - _logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]); - throw new Exception($"Invalid date in row {i}"); + _logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}", + ImporterType, i, data[i][1]); + throw new InvalidOperationException($"Invalid date in row {i}"); } var number = data[i][2].ToString()!; @@ -114,7 +336,8 @@ public class MorskaFk2Importer : MorskaBaseImporter if (string.IsNullOrEmpty(data[i][9]?.ToString()) || !double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value", + ImporterType, i); continue; } @@ -132,20 +355,24 @@ public class MorskaFk2Importer : MorskaBaseImporter } _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -164,6 +391,7 @@ public class MorskaFk2Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs deleted file mode 100644 index b655f15..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ /dev/null @@ -1,151 +0,0 @@ -using System.Globalization; -using Google.Apis.Sheets.v4; -using Microsoft.Extensions.Logging; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; - -namespace DiunaBI.Plugins.Morska.Importers; - -public class MorskaImporter : MorskaBaseImporter -{ - public override string ImporterType => "MorskaImporter"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public MorskaImporter( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - - public override void Import(Layer importWorker) - { - _logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) - { - throw new Exception($"SheetId not found, {importWorker.Name}"); - } - - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) - { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); - } - - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) - { - throw new Exception($"ImportYear not found, {importWorker.Name}"); - } - - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) - { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); - } - - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) - { - throw new Exception($"ImportName not found, {importWorker.Name}"); - } - - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) - { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } - - _logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); - - var layer = new Layer - { - Id = Guid.NewGuid(), - Number = _db.Layers.Count() + 1, - ParentId = importWorker.Id, - Type = LayerType.Import, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; - - var newRecords = new List(); - - try - { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - - _logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); - - if (data != null && data.Count >= 2) - { - for (var i = 0; i < data[1].Count; i++) - { - if (!(data[0][i].ToString()?.Length > 0) || - !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) - { - _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i); - continue; - } - - var record = new Record - { - Id = Guid.NewGuid(), - Code = data[0][i].ToString(), - Value1 = value, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - newRecords.Add(record); - } - } - - _db.Layers.Add(layer); - - SaveRecords(layer.Id, newRecords); - _db.SaveChanges(); - - _logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); - } - catch (Exception e) - { - _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId); - throw; - } - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs new file mode 100644 index 0000000..49a9b5e --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs @@ -0,0 +1,359 @@ +using System.Globalization; +using Google.Apis.Sheets.v4; +using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; + +namespace DiunaBI.Plugins.Morska.Importers; + +public class MorskaStandardImporter : MorskaBaseImporter +{ + public override string ImporterType => "Morska.Import.Standard"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + + private IList>? _cachedSheetData; + private string? _cachedDataKey; + + public MorskaStandardImporter( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Import(Layer importWorker) + { + try + { + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + // Clear cache before import + _cachedSheetData = null; + _cachedDataKey = null; + + // Load configuration from layer records + LoadConfiguration(importWorker); + + // Check if import should be performed + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual import + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + finally + { + // Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; + } + + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) + { + EndDate = endDate; + } + + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) + { + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + // Check date range if configured + if (StartDate.HasValue && EndDate.HasValue) + { + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + // Outside date range - check if imported layer is up to date + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + // No date constraints - always import + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; + } + + try + { + var data = GetSheetData(); + + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + // Check if the number of columns matches + if (data[0].Count != data[1].Count) + { + _logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + for (var i = 0; i < data[1].Count; i++) + { + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); + continue; + } + + var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); + if (existingRecord == null || existingRecord.Value1 != value) + { + _logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}", + ImporterType, data[0][i]); + return false; + } + } + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + _logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); + + var layer = new Layer + { + Id = Guid.NewGuid(), + Number = _db.Layers.Count() + 1, + ParentId = importWorker.Id, + Type = LayerType.Import, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; + + var newRecords = new List(); + + try + { + var data = GetSheetData(); + + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", + ImporterType, data?.Count ?? 0); + + if (data != null && data.Count >= 2) + { + for (var i = 0; i < data[1].Count; i++) + { + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); + continue; + } + + var record = new Record + { + Id = Guid.NewGuid(), + Code = data[0][i].ToString(), + Value1 = value, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + newRecords.Add(record); + } + } + + _db.Layers.Add(layer); + SaveRecords(layer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); + throw; + } + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs new file mode 100644 index 0000000..16d02a0 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs @@ -0,0 +1,496 @@ +using System.Globalization; +using Google.Apis.Sheets.v4; +using Google.Apis.Sheets.v4.Data; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services; +using DiunaBI.Core.Services.Calculations; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT1R1Processor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T1.R1"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + private string? GoogleSheetName { get; set; } + + public MorskaT1R1Processor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + // Load Google Sheet name + GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(GoogleSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for each month + var newRecords = ProcessAllMonths(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List ProcessAllMonths() + { + var newRecords = new List(); + + for (var month = 1; month < 14; month++) + { + // Skip future months (except month 13 which is summary) + if (Year > DateTime.UtcNow.Year || + (Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)) + { + _logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}", + ProcessorType, Year, month); + continue; + } + + _logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}", + ProcessorType, month, Year); + + var monthRecords = ProcessSingleMonth(month); + newRecords.AddRange(monthRecords); + + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}", + ProcessorType, monthRecords.Count, month); + } + + return newRecords; + } + + private List ProcessSingleMonth(int month) + { + var records = new List(); + + // Collect records from all sources + foreach (var source in Sources!) + { + var sourceRecords = GetSourceRecords(source, month); + records.AddRange(sourceRecords); + } + + // Process dynamic codes (calculations) + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(records, month); + records.AddRange(calculatedRecords); + } + + // Create final records with month suffix + var monthRecords = records.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + return monthRecords; + } + + private List GetSourceRecords(Record source, int month) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found"); + } + + var sourceRecords = new List(); + + // Check if there are specific codes configured for this source + var codesRecord = Sources! + .Where(x => x.Code == $"Codes-{source.Desc1}") + .FirstOrDefault(); + + if (codesRecord != null) + { + var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); + sourceRecords.AddRange(dataSource.Records! + .Where(x => codes.Contains(int.Parse(x.Code!)))); + + _logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes", + ProcessorType, source.Desc1, codes.Count); + } + else + { + sourceRecords.AddRange(dataSource.Records!); + + _logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records", + ProcessorType, source.Desc1, dataSource.Records!.Count); + } + + return sourceRecords; + } + + private List ProcessDynamicCodes(List records, int month) + { + var calculatedRecords = new List(); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) + { + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}", + ProcessorType, dynamicCode.Id, month); + continue; + } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}", + ProcessorType, calc.Expression, dynamicCode.Id, month); + continue; + } + + var calculatedRecord = calc.CalculateT1(records); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}", + ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}", + ProcessorType, dynamicCode.Desc1, month); + } + } + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private void UpdateGoogleSheetReport(Guid sourceId) + { + try + { + _logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + + const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; + + // Get processed layer data + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + // Get codes from sheet header + var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute(); + var codesRow = codesResponse.Values[0]; + + // Update monthly data (months 1-12) + UpdateMonthlyData(sheetId, codesRow, processedLayer); + + // Update summary row (month 13) + UpdateSummaryData(sheetId, codesRow, processedLayer); + + // Update timestamps + UpdateTimestamps(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + throw; + } + } + + private void UpdateMonthlyData(string sheetId, IList codesRow, Layer processedLayer) + { + var valueRange = new ValueRange + { + Values = new List>() + }; + + // Process months 1-12 + for (var month = 1; month <= 12; month++) + { + var monthValues = new List(); + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}"); + monthValues.Add(record?.Value1?.ToString() ?? "0"); + } + valueRange.Values.Add(monthValues); + } + + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType); + } + + private void UpdateSummaryData(string sheetId, IList codesRow, Layer processedLayer) + { + var valueRange = new ValueRange + { + Values = new List>() + }; + + // Empty row + var emptyRow = new List(); + for (int i = 0; i < codesRow.Count; i++) + { + emptyRow.Add(""); + } + valueRange.Values.Add(emptyRow); + + // Summary row (month 13) + var summaryValues = new List(); + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + summaryValues.Add(record?.Value1?.ToString() ?? "0"); + } + valueRange.Values.Add(summaryValues); + + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs new file mode 100644 index 0000000..745d1f1 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs @@ -0,0 +1,467 @@ +using System.Globalization; +using System.Text.RegularExpressions; +using DiunaBI.Core.Services; +using Google.Apis.Sheets.v4; +using Google.Apis.Sheets.v4.Data; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT1R3Processor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T1.R3"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + + public MorskaT1R3Processor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$"; + + var dataSources = _db.Layers + .Where(x => !x.IsDeleted && !x.IsCancelled) + .Include(layer => layer.Records!) + .AsNoTracking() + .AsEnumerable() + .Where(x => Regex.IsMatch(x.Name!, pattern)) + .ToList(); + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for pattern: {pattern}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}", + ProcessorType, dataSources.Count, pattern); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); + + foreach (var dataSource in dataSources) + { + var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); + if (monthStr == null || !int.TryParse(monthStr, out var month)) + { + _logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}", + ProcessorType, dataSource.Name); + continue; + } + + _logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}", + ProcessorType, dataSource.Name, month); + + var sourceRecords = ProcessDataSourceRecords(dataSource, month); + newRecords.AddRange(sourceRecords); + + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}", + ProcessorType, sourceRecords.Count, dataSource.Name); + } + + _logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDataSourceRecords(Layer dataSource, int month) + { + var newRecords = new List(); + + foreach (var record in dataSource.Records!) + { + if (record.Value1 == null) + { + _logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null", + ProcessorType, record.Code); + continue; + } + + // Process values for positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(record, i); + if (value == null) + { + continue; + } + + var baseValue = (double)record.Value1!; + var positionValue = (double)value; + var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100; + + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"{record.Code}{month:D2}{i:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = calculatedValue, + Desc1 = record.Desc1 + }; + + newRecords.Add(newRecord); + + _logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}", + ProcessorType, newRecord.Code, newRecord.Value1, record.Code); + } + } + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private void UpdateGoogleSheetReport(Guid sourceId) + { + try + { + _logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}", + ProcessorType, sourceId); + + const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; + + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + // Update sheets for all months + for (var month = 1; month <= 12; month++) + { + UpdateMonthSheet(sheetId, processedLayer, month); + } + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month) + { + var sheetName = ProcessHelper.GetSheetName(month, Year); + + try + { + _logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + + // Get codes from sheet + ValueRange? dataRangeResponse; + try + { + dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); + } + catch (Exception e) + { + _logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}", + ProcessorType, sheetName, e.Message); + return; + } + + if (dataRangeResponse?.Values == null) + { + _logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping", + ProcessorType, sheetName); + return; + } + + // Update data + UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month); + + // Update timestamps + UpdateSheetTimestamps(sheetId, sheetName, processedLayer); + + _logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}", + ProcessorType, sheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + throw; + } + } + + private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList> codeRows, int month) + { + var updateValueRange = new ValueRange + { + Values = new List>() + }; + + foreach (var row in codeRows) + { + if (row.Count == 0) continue; + + var code = row[0].ToString(); + var updateRow = new List(); + + // Process columns C to Q (positions 1-15) + for (var position = 1; position <= 15; position++) + { + var recordCode = $"{code}{month:D2}{position:D2}"; + var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode); + + if (codeRecord?.Value1 != null) + { + updateRow.Add(codeRecord.Value1); + _logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}", + ProcessorType, codeRecord.Value1, recordCode); + } + else + { + updateRow.Add(""); + } + } + + updateValueRange.Values.Add(updateRow); + } + + // Update sheet with new values + var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}", + ProcessorType, updateValueRange.Values.Count, sheetName); + } + + private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, sheetName, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs new file mode 100644 index 0000000..cccb5da --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs @@ -0,0 +1,310 @@ +using DiunaBI.Core.Services; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "T3.MultiSourceCopySelectedCodes"; + + private readonly AppDbContext _db; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private string? Codes { get; set; } + private List? CodesList { get; set; } + + public MorskaT3MultiSourceCopySelectedCodesProcessor( + AppDbContext db, + ILogger logger) + { + _db = db; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load codes + Codes = GetRecordValue(processWorker.Records, "Codes"); + if (string.IsNullOrEmpty(Codes)) + { + throw new InvalidOperationException("Codes record not found"); + } + + // Parse codes list + CodesList = ProcessHelper.ParseCodes(Codes); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}", + ProcessorType, Year, Month, Sources.Count, CodesList.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = new List(); + + foreach (var source in Sources!) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + _logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3", + ProcessorType, Year, Month, source.Desc1); + continue; + } + + dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + } + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + var filteredRecords = allSourceRecords + .Where(x => !string.IsNullOrEmpty(x.Code) && + int.TryParse(x.Code, out var code) && + CodesList!.Contains(code)) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes", + ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count); + + var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateCopiedRecord(Record sourceRecord) + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = sourceRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(sourceRecord, i); + ProcessHelper.SetValue(newRecord, i, value); + } + + _logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32", + ProcessorType, newRecord.Code); + + return newRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs new file mode 100644 index 0000000..b309ab2 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs @@ -0,0 +1,275 @@ +using DiunaBI.Core.Services; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary"; + + private readonly AppDbContext _db; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + + public MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor( + AppDbContext db, + ILogger logger) + { + _db = db; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}", + ProcessorType, Year); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}", + ProcessorType, Year); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = new List(); + + for (var month = 1; month <= 12; month++) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource != null) + { + dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); + } + } + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for year {Year}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) + { + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0); + ProcessHelper.SetValue(processedRecord, position, totalValue); + } + + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); + } + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs new file mode 100644 index 0000000..e5340e4 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs @@ -0,0 +1,340 @@ +using DiunaBI.Core.Services; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services.Calculations; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3MultiSourceSummaryProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T3.MultiSourceSummary"; + + private readonly AppDbContext _db; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + + public MorskaT3MultiSourceSummaryProcessor( + AppDbContext db, + ILogger logger) + { + _db = db; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault()) + .OfType() + .ToList(); + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); + + foreach (var baseCode in baseCodes) + { + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"9{baseCode}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Sum values from all sources for positions 1-32 + for (var i = 1; i < 33; i++) + { + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); + } + + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); + } + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) + { + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; + } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; + } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); + } + } + + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs new file mode 100644 index 0000000..0b8642b --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs @@ -0,0 +1,344 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services; +using DiunaBI.Core.Services.Calculations; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3MultiSourceYearSummaryProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary"; + + private readonly AppDbContext _db; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + + public MorskaT3MultiSourceYearSummaryProcessor( + AppDbContext db, + ILogger logger) + { + _db = db; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes with validation) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault()) + .OfType() + .ToList(); + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for year {Year}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); + + foreach (var baseCode in baseCodes) + { + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"9{baseCode}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Validation record for double-checking calculations + var validationRecord = new Record(); + + // Sum values from all sources for positions 1-32 with validation + for (var i = 1; i < 33; i++) + { + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); + + // Validation calculation (identical to main calculation) + var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(validationRecord, i, validationValue); + + // Validate that both calculations match + var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!); + if (difference > 0.01) + { + throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})"); + } + } + + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); + } + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) + { + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; + } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; + } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); + } + } + + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs new file mode 100644 index 0000000..15e7602 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs @@ -0,0 +1,340 @@ +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3SingleSourceProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T3.SingleSource"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayerName { get; set; } + private string? Source { get; set; } + private Layer? SourceImportWorker { get; set; } + + public MorskaT3SingleSourceProcessor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year and month + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer name + SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayerName)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source name + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayerName, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + // Find source import worker + SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled); + if (SourceImportWorker == null) + { + errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found"); + } + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled) + .OrderBy(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + _logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records", + ProcessorType, allRecords.Count); + + foreach (var baseRecord in dataSources.Last().Records!) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + ProcessDailyValues(processedRecord, codeRecords); + + newRecords.Add(processedRecord); + } + + _logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void ProcessDailyValues(Record processedRecord, List codeRecords) + { + var lastDayInMonth = DateTime.DaysInMonth(Year, Month); + + // Day 1 - first value for the month + var firstVal = codeRecords + .Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1)) + .MaxBy(x => x.CreatedAt)?.Value1 ?? 0; + + ProcessHelper.SetValue(processedRecord, 1, firstVal); + var previousValue = firstVal; + + // Days 2 to last-1 - daily differences + for (var i = 2; i < lastDayInMonth; i++) + { + var dayVal = codeRecords + .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (dayVal == null) + { + ProcessHelper.SetValue(processedRecord, i, 0); + } + else + { + var processedVal = dayVal - previousValue; + ProcessHelper.SetValue(processedRecord, i, processedVal); + previousValue = (double)dayVal; + } + } + + // Last day - special handling + var lastVal = codeRecords + .Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth)) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (lastVal == null) + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); + } + else + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); + } + + // Copy last value to position 32 + var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; + ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + + _logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}", + ProcessorType, processedRecord.Code, valueToCopy); + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id); + + if (isNew) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs new file mode 100644 index 0000000..2280cc7 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs @@ -0,0 +1,288 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT3SourceYearSummaryProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T3.SourceYearSummary"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + + public MorskaT3SourceYearSummaryProcessor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = new List(); + + for (var month = 1; month <= 12; month++) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource != null) + { + dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); + } + } + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}", + ProcessorType, dataSources.Count, Year, Source); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) + { + var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = baseRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) + { + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position)); + ProcessHelper.SetValue(processedRecord, position, totalValue); + } + + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); + } + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs new file mode 100644 index 0000000..81d49b1 --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs @@ -0,0 +1,600 @@ +using System.Globalization; +using Google.Apis.Sheets.v4; +using Google.Apis.Sheets.v4.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using DiunaBI.Core.Services; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT4R2Processor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T4.R2"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private string? LayerName { get; set; } + private string? ReportSheetName { get; set; } + private string? InvoicesSheetName { get; set; } + + public MorskaT4R2Processor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load layer name + LayerName = GetRecordValue(processWorker.Records, "LayerName"); + if (string.IsNullOrEmpty(LayerName)) + { + throw new InvalidOperationException("LayerName record not found"); + } + + // Load report sheet name + ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(ReportSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + // Load invoices sheet name + InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices"); + if (string.IsNullOrEmpty(InvoicesSheetName)) + { + throw new InvalidOperationException("GoogleSheetName-Invoices record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}", + ProcessorType, Year, Sources.Count, LayerName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required"); + if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required"); + if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for all sources + var newRecords = ProcessSources(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheets reports + UpdateReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-{LayerName}"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List ProcessSources() + { + var newRecords = new List(); + + foreach (var source in Sources!) + { + _logger.LogDebug("{ProcessorType}: Processing source {Source}", + ProcessorType, source.Desc1); + + var sourceCodes = GetSourceCodes(source); + var sourceRecords = ProcessSourceData(source, sourceCodes); + newRecords.AddRange(sourceRecords); + + _logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records", + ProcessorType, source.Desc1, sourceRecords.Count); + } + + _logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List GetSourceCodes(Record source) + { + var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}"); + if (string.IsNullOrEmpty(rawSourceCodes)) + { + return new List(); + } + + return ProcessHelper.ParseCodes(rawSourceCodes); + } + + private List ProcessSourceData(Record source, List sourceCodes) + { + var sourceRecords = new List(); + var lastSourceCodes = new List(); + + // Process monthly data (1-12) + for (var month = 1; month <= 12; month++) + { + var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes); + sourceRecords.AddRange(monthRecords); + } + + // Process year summary (month 13) + var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes); + sourceRecords.AddRange(yearSummaryRecords); + + return sourceRecords; + } + + private List ProcessMonthData(Record source, List sourceCodes, int month, List lastSourceCodes) + { + var monthRecords = new List(); + + if (IsDataAvailableForMonth(month)) + { + var dataSource = GetMonthDataSource(source, month); + if (dataSource != null) + { + lastSourceCodes.Clear(); + lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!)); + + var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes); + var processedRecords = CreateMonthRecords(filteredRecords, source, month); + monthRecords.AddRange(processedRecords); + + _logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records", + ProcessorType, month, source.Desc1, processedRecords.Count); + } + else + { + _logger.LogWarning("{ProcessorType}: Data source {DataSource} not found", + ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T"); + } + } + else + { + // Future months - create zero value records (except for FK2) + if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0) + { + var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month); + monthRecords.AddRange(futureRecords); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}", + ProcessorType, futureRecords.Count, month); + } + } + + return monthRecords; + } + + private bool IsDataAvailableForMonth(int month) + { + return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year; + } + + private Layer? GetMonthDataSource(Record source, int month) + { + return _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + } + + private List FilterRecords(IEnumerable records, List sourceCodes) + { + return records + .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) + .ToList(); + } + + private List CreateMonthRecords(List filteredRecords, Record source, int month) + { + return filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, + Desc1 = x.Desc1 + }).ToList(); + } + + private List CreateFutureMonthRecords(List lastSourceCodes, List sourceCodes, int month) + { + return lastSourceCodes + .Where(x => sourceCodes.Contains(int.Parse(x))) + .Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = 0 + }).ToList(); + } + + private List ProcessYearSummaryData(Record source, List sourceCodes) + { + var dataSourceSum = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSourceSum == null) + { + _logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found", + ProcessorType, $"{Year}/13-{source.Desc1}-T3"); + return new List(); + } + + var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes); + var yearSummaryRecords = filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}13", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}", + ProcessorType, yearSummaryRecords.Count, source.Desc1); + + return yearSummaryRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private void UpdateReport(Guid sourceId) + { + try + { + _logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}", + ProcessorType, sourceId); + + const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; + + var processedLayer = GetProcessedLayerData(sourceId); + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + var codesRow = GetCodesFromSheet(sheetId); + + UpdateMonthlyData(sheetId, processedLayer, codesRow); + UpdateYearSummary(sheetId, processedLayer, codesRow); + UpdateTimestamps(sheetId, processedLayer); + UpdateInvoicesData(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private Layer? GetProcessedLayerData(Guid sourceId) + { + return _db.Layers + .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + } + + private IList GetCodesFromSheet(string sheetId) + { + var request = _googleSheetValues.Get(sheetId, "C4:Z4"); + var response = request.Execute(); + return response.Values[0]; + } + + private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList codesRow) + { + const int startRow = 6; + + for (var month = 1; month <= 12; month++) + { + var values = new List(); + var monthStr = month < 10 ? $"0{month}" : month.ToString(); + + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}"); + values.Add(record?.Value1?.ToString() ?? "0"); + } + + var valueRange = new ValueRange + { + Values = new List> { values } + }; + + var row = (startRow + month).ToString(); + var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet", + ProcessorType, month); + } + } + + private void UpdateYearSummary(string sheetId, Layer processedLayer, IList codesRow) + { + const int startRow = 6; + var valuesSum = new List(); + var emptyRow = new List(); + + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + emptyRow.Add(""); + valuesSum.Add(record?.Value1?.ToString() ?? "0"); + } + + // Insert empty row before sum + var rowEmpty = (startRow + 13).ToString(); + var valueRangeEmpty = new ValueRange + { + Values = new List> { emptyRow } + }; + var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); + updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateEmpty.Execute(); + + // Update sum row + var rowSum = (startRow + 14).ToString(); + var valueRangeSum = new ValueRange + { + Values = new List> { valuesSum } + }; + var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}"); + updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateSum.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + // Update UTC time + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + var valueRangeUtcTime = new ValueRange + { + Values = new List> { new List { timeUtc } } + }; + var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1"); + updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTimeUtc.Execute(); + + // Update Warsaw time + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + var valueRangeWarsawTime = new ValueRange + { + Values = new List> { new List { timeWarsaw } } + }; + var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2"); + updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTimeWarsaw.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } + + private void UpdateInvoicesData(string sheetId, Layer processedLayer) + { + var invoices = processedLayer.Records! + .Where(x => x.Code!.Length == 12) + .OrderByDescending(x => x.Code) + .ToList(); + + var invoicesValues = new List>(); + var cleanUpValues = new List>(); + + foreach (var invoice in invoices) + { + var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) + .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); + + var invoiceRow = new List + { + invoiceDate, "", invoice.Desc1!, invoice.Value1! + }; + invoicesValues.Add(invoiceRow); + + var cleanupRow = new List { "", "", "", "" }; + cleanUpValues.Add(cleanupRow); + } + + // Clear existing data + var cleanupValueRange = new ValueRange { Values = cleanUpValues }; + var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + cleanupInvoices.Execute(); + + // Update with new data + var invoicesValueRange = new ValueRange { Values = invoicesValues }; + var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateInvoices.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet", + ProcessorType, invoices.Count); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs new file mode 100644 index 0000000..c4bd66e --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs @@ -0,0 +1,292 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT4SingleSourceProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T4.SingleSource"; + + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + + public MorskaT4SingleSourceProcessor( + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) + { + _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data source + var dataSource = GetDataSource(sourceImportWorker); + + // Process records (simple copy) + var newRecords = ProcessRecords(dataSource); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + + if (sourceImportWorker == null) + { + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); + } + + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private Layer GetDataSource(Layer sourceImportWorker) + { + var dataSource = _db.Layers + .Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled) + .Include(x => x.Records) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}"); + } + + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + + return dataSource; + } + + private List ProcessRecords(Layer dataSource) + { + if (dataSource.Records == null || dataSource.Records.Count == 0) + { + _logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType); + return new List(); + } + + var newRecords = dataSource.Records.Select(record => new Record + { + Id = Guid.NewGuid(), + Code = record.Code, + Desc1 = record.Desc1, + Value1 = record.Value1, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs new file mode 100644 index 0000000..0d84c0c --- /dev/null +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs @@ -0,0 +1,320 @@ +using DiunaBI.Core.Services; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; +using Microsoft.Extensions.Logging; + +namespace DiunaBI.Plugins.Morska.Processors; + +public class MorskaT5LastValuesProcessor : MorskaBaseProcessor +{ + public override string ProcessorType => "Morska.Process.T5.LastValues"; + + private readonly AppDbContext _db; + private readonly ILogger _logger; + + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + + public MorskaT5LastValuesProcessor( + AppDbContext db, + ILogger logger) + { + _db = db; + _logger = logger; + } + + public override void Process(Layer processWorker) + { + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(sourceImportWorker); + + // Process records (get last values for each code) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + + if (sourceImportWorker == null) + { + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); + } + + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { + var processedLayer = _db.Layers + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (processedLayer == null) + { + processedLayer = new Layer + { + Id = Guid.NewGuid(), + Type = LayerType.Processed, + ParentId = processWorker.Id, + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { + processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + + return processedLayer; + } + + private List GetDataSources(Layer sourceImportWorker) + { + var dataSources = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == sourceImportWorker.Id && + !x.IsDeleted && !x.IsCancelled) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}", + ProcessorType, dataSources.Count, sourceImportWorker.Name); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var codes = allRecords.Select(x => x.Code).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records", + ProcessorType, codes.Count, allRecords.Count); + + foreach (var code in codes) + { + var lastRecord = allRecords + .Where(x => x.Code == code) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + + if (lastRecord == null) continue; + + var processedRecord = CreateProcessedRecord(lastRecord); + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}", + ProcessorType, code, lastRecord.CreatedAt); + } + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateProcessedRecord(Record lastRecord) + { + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = lastRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(lastRecord, i); + if (value != null) + { + ProcessHelper.SetValue(processedRecord, i, value); + } + } + + return processedRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { + _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } + else + { + _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + + SaveRecords(processedLayer.Id, newRecords); + _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + // Remove existing records for this layer + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); + } + + // Add new records + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs deleted file mode 100644 index d9d87fa..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ /dev/null @@ -1,284 +0,0 @@ -using System.Globalization; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services; -using DiunaBI.Core.Services.Calculations; -using Microsoft.Extensions.Logging; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T1R1Processor : MorskaBaseProcessor -{ - public override string ProcessorType => "T1.R1"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T1R1Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) - .ToList(); - - var newRecords = new List(); - - for (var month = 1; month < 14; month++) - { - if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))) - { - continue; - } - - var records = new List(); - foreach (var source in sources) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3") - ).Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - if (dataSource == null) - { - throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found."); - } - - var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault(); - if (codesRecord != null) - { - var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); - records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!)))); - } - else - { - records.AddRange(dataSource.Records!); - } - } - - if (dynamicCodes != null) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - records.Add(calc.CalculateT1(records)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - } - - newRecords.AddRange(records.Select(x => new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - } - )); - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (sheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - UpdateReport(processedLayer.Id, sheetName); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private void UpdateReport(Guid sourceId, string sheetName) - { - const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; - var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4"); - var response = request.Execute(); - - var r1 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - var codesRow = response.Values[0]; - - var valueRange = new ValueRange - { - Values = new List>() - }; - - for (var i = 1; i <= 12; i++) - { - var values = new List(); - foreach (string code in codesRow) - { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } - } - valueRange.Values.Add(values); - } - - // sum - var valuesSum = new List(); - var emptyRow = new List(); - foreach (string code in codesRow) - { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); - emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } - } - - valueRange.Values.Add(emptyRow); - valueRange.Values.Add(valuesSum); - - var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20"); - update.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - update.Execute(); - - // update time - var timeUtc = new List - { - r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange - { - Values = new List>() - }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); - - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs deleted file mode 100644 index 6a6f633..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ /dev/null @@ -1,227 +0,0 @@ -using System.Globalization; -using System.Text.RegularExpressions; -using DiunaBI.Core.Services; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using Microsoft.Extensions.Logging; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T1R3Processor : MorskaBaseProcessor -{ - public override string ProcessorType => "T1.R3"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T1R3Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - var newRecords = new List(); - - string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$"; - var dataSources = _db.Layers - .Where(x => !x.IsDeleted && !x.IsCancelled) - .Include(layer => layer.Records!) - .AsNoTracking() - .AsEnumerable() - .Where(x => Regex.IsMatch(x.Name!, pattern)) - .ToList(); - - foreach (var dataSource in dataSources) - { - var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); - if (month == null) - { - throw new Exception($"Month not found: {dataSource.Name}"); - } - - foreach (var record in dataSource.Records!) - { - if (record.Value1 == null) continue; - for (var i = 1; i < 33; i++) - { - if (ProcessHelper.GetValue(record, i) == null) continue; - - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{record.Code}{month}{i:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = i == 1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100, - Desc1 = record.Desc1 - }; - - newRecords.Add(newRecord); - } - } - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - UpdateReport(processedLayer.Id, year); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private void UpdateReport(Guid sourceId, int year) - { - const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; - - var r3 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - for (var i = 1; i <= 12; i++) - { - var sheetName = ProcessHelper.GetSheetName(i, year); - ValueRange? dataRangeResponse; - try - { - dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); - } - catch - { - continue; // Sheet not exist - } - - if (dataRangeResponse == null) continue; // Sheet not exist - var data = dataRangeResponse.Values; - - var updateValueRange = new ValueRange - { - Values = new List>() - }; - - foreach (var row in data) - { - if (row.Count == 0) continue; - var code = row[0].ToString(); - - var updateRow = new List(); - - for (var j = 1; j < 16; j++) - { - var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}"); - if (codeRecord is { Value1: not null }) - { - updateRow.Add(codeRecord.Value1); - } - else - { - updateRow.Add(""); - } - } - - updateValueRange.Values.Add(updateRow); - } - - dataRangeResponse.Values = data; - var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); - update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - update.Execute(); - - // update time - var timeUtc = new List - { - r3!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r3.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange - { - Values = new List>() - }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); - - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); - } - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs deleted file mode 100644 index c35f36c..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs +++ /dev/null @@ -1,131 +0,0 @@ -using DiunaBI.Core.Services; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using Microsoft.Extensions.Logging; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.MultiSourceCopySelectedCodes"; - - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public T3MultiSourceCopySelectedCodesProcessor( - AppDbContext db, - ILogger logger) - { - _db = db; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) - { - throw new Exception("Source record not found"); - } - var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; - if (codes == null) - { - throw new Exception("Codes record not found"); - } - - var codesList = ProcessHelper.ParseCodes(codes); - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dataSources = sources.Select(source => _db.Layers - .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) - .Include(x => x.Records).AsNoTracking() - .FirstOrDefault()) - .OfType() - .ToList(); - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - - var newRecords = dataSources - .SelectMany(x => x.Records!) - .Where(x => codesList.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); - } - return newRecord; - }) - .ToList(); - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs deleted file mode 100644 index d54728b..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs +++ /dev/null @@ -1,131 +0,0 @@ -using DiunaBI.Core.Services; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using Microsoft.Extensions.Logging; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary"; - - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public T3MultiSourceCopySelectedCodesYearSummaryProcessor( - AppDbContext db, - ILogger logger) - { - _db = db; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); - - var dataSources = new List(); - - for (var i = 1; i < 13; i++) - { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-AB-T3")) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource); - } - } - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (var baseRecord in dataSources.Last().Records!) - { - var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - - newRecords.Add(processedRecord); - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - } - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs deleted file mode 100644 index b4a7cb7..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ /dev/null @@ -1,182 +0,0 @@ -using DiunaBI.Core.Services; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services.Calculations; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.MultiSourceSummary"; - - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public T3MultiSourceSummaryProcessor( - AppDbContext db, - ILogger logger) - { - _db = db; - _logger = logger; - } - - public override void Process(Layer processWorker) - { - _logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault()) - .OfType() - .ToList(); - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); - - foreach (var baseCode in baseCodes) - { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"9{baseCode}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - newRecords.Add(processedRecord); - } - - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - _logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs deleted file mode 100644 index 90f7361..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ /dev/null @@ -1,198 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services; -using DiunaBI.Core.Services.Calculations; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.MultiSourceYearSummary"; - - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public T3MultiSourceYearSummaryProcessor( - AppDbContext db, - ILogger logger) - { - _db = db; - _logger = logger; - } - - public override void Process(Layer processWorker) - { - _logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault()) - .OfType() - .ToList(); - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - var allRecords = dataSources - .SelectMany(x => x.Records!).ToList(); - var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); - - foreach (var baseCode in baseCodes) - { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); - var codeRecordsValidation = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"9{baseCode}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - var validationRecord = new Record(); - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - - ProcessHelper.SetValue(validationRecord, i, - codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); - - if ( - double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - - ProcessHelper.GetValue(validationRecord, i))!) > 0.01) - { - throw new Exception($"ValidationError: Code {baseCode}, " + - $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + - $"{ProcessHelper.GetValue(validationRecord, i)})"); - } - } - newRecords.Add(processedRecord); - } - - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - _logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs deleted file mode 100644 index a4c9c9d..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ /dev/null @@ -1,177 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services; -using Microsoft.Extensions.Logging; -using Google.Apis.Sheets.v4; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3SingleSourceProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.SingleSource"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T3SingleSourceProcessor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("SourceLayer record not found"); - } - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) - { - throw new Exception("SourceImportWorkerL layer not found"); - } - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - var newRecords = new List(); - - var dataSources = _db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - if (dataSources.Count == 0) - { - throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); - } - - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (var baseRecord in dataSources.Last().Records!) - { - var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - var lastDayInMonth = DateTime.DaysInMonth(year, month); - //day 1 - var firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0; - ProcessHelper.SetValue(processedRecord, 1, firstVal); - var previousValue = firstVal; - //days 2-29/30 - for (var i = 2; i < lastDayInMonth; i++) - { - var dayVal = codeRecords - .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1; - if (dayVal == null) - { - ProcessHelper.SetValue(processedRecord, i, 0); - } - else - { - var processedVal = dayVal - previousValue; - ProcessHelper.SetValue(processedRecord, i, processedVal); - previousValue = (double)dayVal; - } - } - //last day - var lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1; - - if (lastVal == null) - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); - } - else - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); - } - - // copy last value - var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; - ProcessHelper.SetValue(processedRecord, 32, valueToCopy); - - newRecords.Add(processedRecord); - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - } - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs deleted file mode 100644 index 85354e9..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ /dev/null @@ -1,147 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using Google.Apis.Sheets.v4; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T3SourceYearSummaryProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T3.SourceYearSummary"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T3SourceYearSummaryProcessor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - - public override void Process(Layer processWorker) - { - _logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); - - var dataSources = new List(); - for (var i = 1; i < 13; i++) - { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-{source}-T3")) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSource != null) - { - dataSources.Add(dataSource); - } - } - - if (dataSources.Count == 0) - { - throw new Exception("DataSources are empty"); - } - - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); - - foreach (var baseRecord in dataSources.Last().Records!) - { - var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = baseRecord.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); - } - newRecords.Add(processedRecord); - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - _logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs deleted file mode 100644 index edb9e73..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ /dev/null @@ -1,386 +0,0 @@ -using System.Globalization; -using Google.Apis.Sheets.v4; -using Google.Apis.Sheets.v4.Data; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using DiunaBI.Core.Services; - - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T4R2Processor : MorskaBaseProcessor -{ - public override string ProcessorType => "T4.R2"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T4R2Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - - public override void Process(Layer processWorker) - { - _logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) - { - throw new Exception("Source record not found"); - } - - var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; - if (layerName == null) - { - throw new Exception("LayerName record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - var newRecords = new List(); - - foreach (var source in sources) - { - var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") - ?.Desc1; - var sourceCodes = new List(); - if (rawSourceCodes != null) - { - sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); - } - - List lastSourceCodes = []; - - for (var month = 1; month <= 12; month++) - { - if ((year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || year < DateTime.UtcNow.Year) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSource != null) - { - lastSourceCodes = dataSource.Records!.Select(x => x.Code!).ToList(); - var news = dataSource.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, - Desc1 = x.Desc1 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id); - } - } - else - { - //0 values for future months - if (source.Desc1 == "FK2" || lastSourceCodes.Count <= 0) continue; - var news = lastSourceCodes - .Where(x => sourceCodes.Contains(int.Parse(x))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = 0, - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - } - - // year summary - var dataSourceSum = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSourceSum != null) - { - var news = dataSourceSum.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}13", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id); - } - } - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (reportSheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - var invoicesSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName-Invoices")?.Desc1; - if (invoicesSheetName == null) - { - throw new Exception("GoogleSheetName-Invoices record not found"); - } - - UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName); - - _logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName) - { - const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; - var request = _googleSheetValues.Get(sheetId, "C4:Z4"); - var response = request.Execute(); - - var r2 = _db.Layers - .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - const int startRow = 6; - - var codesRow = response.Values[0]; - for (var i = 1; i <= 12; i++) - { - var values = new List(); - var month = i < 10 ? $"0{i}" : i.ToString(); - var row = (startRow + i).ToString(); - foreach (string code in codesRow) - { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}{month}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } - } - var valueRange = new ValueRange - { - Values = new List> { values } - }; - var update = _googleSheetValues.Update(valueRange, sheetId, $"{reportSheetName}!C{row}:XZ{row}"); - update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - update.Execute(); - } - - // sum - var valuesSum = new List(); - var emptyRow = new List(); - var rowEmpty = (startRow + 13).ToString(); - var rowSum = (startRow + 14).ToString(); - foreach (string code in codesRow) - { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); - emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } - } - // insert empty row before sum - var valueRangeEmpty = new ValueRange - { - Values = new List> { emptyRow } - }; - var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{reportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); - updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateEmpty.Execute(); - - var valueRangeSum = new ValueRange - { - Values = new List> { valuesSum } - }; - var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{reportSheetName}!C{rowSum}:XZ{rowSum}"); - updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateSum.Execute(); - - // update time - var timeUtc = new List - { - r2!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeUtcTime = new ValueRange - { - Values = new List> { timeUtc } - }; - var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{reportSheetName}!G1"); - updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); - - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r2.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeWarsawTime = new ValueRange - { - Values = new List> { timeWarsaw } - }; - var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{reportSheetName}!G2"); - updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeWarsaw.Execute(); - - //invoices - - var invoices = r2.Records!.Where(x => x.Code!.Length == 12) - .OrderByDescending(x => x.Code); - - var invoicesValues = new List>(); - var cleanUpValues = new List>(); - foreach (var invoice in invoices) - { - var invoiceDate = - DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) - .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); - var invoiceRow = new List - { - invoiceDate, - "", - invoice.Desc1!, - invoice.Value1! - }; - invoicesValues.Add(invoiceRow); - - var cleanupRow = new List - { - "", "", "", "" - }; - cleanUpValues.Add(cleanupRow); - } - - - var cleanupValueRange = new ValueRange { Values = cleanUpValues }; - var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - cleanupInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - cleanupInvoices.Execute(); - - - var invoicesValueRange = new ValueRange { Values = invoicesValues }; - var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - updateInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateInvoices.Execute(); - - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs deleted file mode 100644 index 1d44314..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ /dev/null @@ -1,138 +0,0 @@ -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using Microsoft.Extensions.Logging; -using Google.Apis.Sheets.v4; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T4SingleSourceProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T4.SingleSource"; - - private readonly AppDbContext _db; - private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; - - public T4SingleSourceProcessor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) - { - _db = db; - _googleSheetValues = googleSheetValues; - _logger = logger; - } - - public override void Process(Layer processWorker) - { - _logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) - { - throw new Exception("SourceLayer record not found"); - } - - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) - { - throw new Exception("SourceImportWorker layer not found"); - } - - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T4"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dataSource = _db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .AsNoTracking() - .FirstOrDefault(); - - if (dataSource == null) - { - throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); - } - - var newRecords = dataSource.Records!.Select(record => new Record - { - Id = Guid.NewGuid(), - Code = record.Code, - Desc1 = record.Desc1, - Value1 = record.Value1, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }).ToList(); - - if (isNew) - { - _db.Layers.Add(processedLayer); - } - else - { - _db.Layers.Update(processedLayer); - } - - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - - _logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - } - - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs deleted file mode 100644 index 5b3bc56..0000000 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ /dev/null @@ -1,126 +0,0 @@ -using DiunaBI.Core.Services; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Database.Context; -using Microsoft.Extensions.Logging; - -namespace DiunaBI.Plugins.Morska.Processors; - -public class T5LastValuesProcessor : MorskaBaseProcessor -{ - public override string ProcessorType => "T5.LastValues"; - - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public T5LastValuesProcessor( - AppDbContext db, - ILogger logger) - { - _db = db; - _logger = logger; - } - public override void Process(Layer processWorker) - { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) throw new Exception("SourceLayer record not found"); - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) throw new Exception("SourceImportWorker layer not found"); - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) throw new Exception("Source record not found"); - - var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .FirstOrDefault(); - - var isNew = false; - if (processedLayer == null) - { - isNew = true; - processedLayer = new Layer - { - Id = Guid.NewGuid(), - Type = LayerType.Processed, - ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 - }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T5"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; - processedLayer.ModifiedAt = DateTime.UtcNow; - } - - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); - - var dataSources = _db.Layers - .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) - .OrderByDescending(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - - if (dataSources.Count == 0) throw new Exception($"DataSource is empty, {sourceImportWorker.Name}"); - - var codes = dataSources.SelectMany(x => x.Records!).Select(x => x.Code).Distinct().ToList(); - - foreach (var code in codes) - { - var lastRecord = dataSources.SelectMany(x => x.Records!).Where(x => x.Code == code).OrderByDescending(x => x.CreatedAt).FirstOrDefault(); - if (lastRecord == null) continue; - - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - for (var i = 1; i < 33; i++) - { - if (ProcessHelper.GetValue(lastRecord, i) != null) - { - ProcessHelper.SetValue(processedRecord, i, ProcessHelper.GetValue(lastRecord, i)); - } - } - - newRecords.Add(processedRecord); - } - - if (isNew) - _db.Layers.Add(processedLayer); - else - _db.Layers.Update(processedLayer); - SaveRecords(processedLayer.Id, newRecords); - _db.SaveChanges(); - } - private void SaveRecords(Guid layerId, ICollection records) - { - var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = layerId; - _db.Records.Add(record); - } - - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs index 7ae6717..3bb10b0 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs @@ -6,8 +6,8 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; using System.Text; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; - +using DiunaBI.Core.Database.Context; + namespace DiunaBI.WebAPI.Controllers; [ApiController] diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs index 10ad60f..27cf379 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs @@ -3,7 +3,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Models; namespace DiunaBI.WebAPI.Controllers; diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 0a35f32..af9bc21 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -5,9 +5,9 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; -using Google.Cloud.Firestore; +using DiunaBI.Core.Interfaces; namespace DiunaBI.WebAPI.Controllers; @@ -20,15 +20,15 @@ public class LayersController : Controller private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; - private readonly ILogger _logger; + private readonly ILogger _logger; public LayersController( AppDbContext db, - SpreadsheetsResource.ValuesResource? googleSheetValues, + SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, - ILogger logger + ILogger logger ) { _db = db; @@ -59,7 +59,7 @@ public class LayersController : Controller .OrderByDescending(x => x.Number) .Skip(start).Take(limit).AsNoTracking().ToList(); - _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", + _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", result.Count, name, type); return Ok(result); @@ -228,7 +228,7 @@ public class LayersController : Controller _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); export.Export(layer); _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); - + return Ok(true); } catch (Exception e) @@ -238,51 +238,6 @@ public class LayersController : Controller } } - [HttpGet] - [Route("AutoImportWithQueue/{apiKey}")] - [AllowAnonymous] - public IActionResult AutoImportWithQueue(string apiKey) - { - if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) - { - _logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey); - return Unauthorized(); - } - - var importWorkerLayers = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - ) - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - - if (importWorkerLayers.Count == 0) - { - _logger.LogInformation("AutoImportQueue: No layers to import"); - return Ok(); - } - - _logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count); - - foreach (var importWorker in importWorkerLayers) - { - try - { - // Queue job implementation would go here - _logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); - } - catch (Exception e) - { - _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", - importWorker.Name, importWorker.Id); - } - } - return Ok(); - } - [HttpGet] [Route("ProcessQueue/{apiKey}")] [AllowAnonymous] @@ -328,7 +283,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", + _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", nameFilter, importWorkerLayers.Count); try @@ -345,8 +300,8 @@ public class LayersController : Controller { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; - - _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", + + _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", importWorker.Name, type, source); if (source == "DataInbox" && type == "Import-D3") @@ -357,8 +312,8 @@ public class LayersController : Controller throw new Exception("MorskaD3 importer not found"); } d3Importer.Import(importWorker); - - _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", + + _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); continue; } @@ -374,7 +329,7 @@ public class LayersController : Controller d1importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -387,7 +342,7 @@ public class LayersController : Controller fk2importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -406,7 +361,7 @@ public class LayersController : Controller var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - + if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = _pluginManager.GetImporter("MorskaImporter"); @@ -417,7 +372,7 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else if (IsImportedLayerUpToDate(importWorker) == false) @@ -430,12 +385,12 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", + _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else { - _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", + _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", importWorker.Name, importWorker.Id); } break; @@ -443,7 +398,7 @@ public class LayersController : Controller } catch (Exception e) { - _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", + _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } @@ -505,7 +460,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", + _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", type, processWorkerLayers.Count); foreach (var processWorker in processWorkerLayers) @@ -513,12 +468,12 @@ public class LayersController : Controller try { ProcessLayer(processWorker); - _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } catch (Exception e) { - _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } } @@ -673,7 +628,9 @@ public class LayersController : Controller } } } - + [HttpGet] + [Route("CheckProcessors")] + [AllowAnonymous] private static void WriteToConsole(params string[] messages) { @@ -734,7 +691,7 @@ public class LayersController : Controller var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", data[0][i].ToString(), importWorker.Name); isUpToDate = false; continue; @@ -752,12 +709,12 @@ public class LayersController : Controller continue; } - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", record.Code, importWorker.Name); isUpToDate = false; } - _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", + _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; diff --git a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj index 6f45a65..6f4c3de 100644 --- a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj +++ b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj @@ -21,7 +21,6 @@ - diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 0d62ca9..440e6c5 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -1,31 +1,33 @@ -using Google.Apis.Auth.OAuth2; -using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; using System.IdentityModel.Tokens.Jwt; using System.Reflection; using System.Text; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using Google.Apis.Sheets.v4; using Serilog; +using DiunaBI.Core.Interfaces; var builder = WebApplication.CreateBuilder(args); -// ✅ DODAJ SERILOG CONFIGURATION -builder.Host.UseSerilog((context, configuration) => +if (builder.Environment.IsProduction()) { - configuration - .ReadFrom.Configuration(context.Configuration) - .Enrich.FromLogContext() - .Enrich.WithProperty("Application", "DiunaBI") - .Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown") - .Enrich.WithEnvironmentName() - .Enrich.WithMachineName(); -}); + builder.Host.UseSerilog((context, configuration) => + { + configuration + .ReadFrom.Configuration(context.Configuration) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", "DiunaBI") + .Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown") + .Enrich.WithEnvironmentName() + .Enrich.WithMachineName(); + }); +} var connectionString = builder.Configuration.GetConnectionString("SQLDatabase"); + builder.Services.AddDbContext(x => { x.UseSqlServer(connectionString); @@ -50,7 +52,6 @@ builder.Services.AddCors(options => builder.Services.AddControllers(); - builder.Services.AddAuthentication(options => { options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; @@ -66,23 +67,22 @@ builder.Services.AddAuthentication(options => ValidateIssuerSigningKey = true, IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!)) }; - }); -builder.Services.AddAuthentication(); -// Zarejestruj Google Sheets dependencies +// Google Sheets dependencies +Console.WriteLine("Adding Google Sheets dependencies..."); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(provider => { var googleSheetsHelper = provider.GetRequiredService(); var valuesResource = googleSheetsHelper.Service?.Spreadsheets.Values; - + if (valuesResource == null) { throw new InvalidOperationException("Google Sheets Service is not initialized properly"); } - + return valuesResource; }); @@ -90,32 +90,45 @@ builder.Services.AddSingleton(); var app = builder.Build(); -app.UseSerilogRequestLogging(options => +if (app.Environment.IsProduction()) { - options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; - options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + app.UseSerilogRequestLogging(options => { - diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); - diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); - - var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); - if (!string.IsNullOrEmpty(userAgent)) + options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => { - diagnosticContext.Set("UserAgent", userAgent); - } - - diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); - diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); - }; -}); + diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); + diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); + var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); + if (!string.IsNullOrEmpty(userAgent)) + { + diagnosticContext.Set("UserAgent", userAgent); + } + + diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); + diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); + }; + }); +} + +// Plugin initialization var pluginManager = app.Services.GetRequiredService(); var executablePath = Assembly.GetExecutingAssembly().Location; var executableDir = Path.GetDirectoryName(executablePath)!; var pluginsPath = Path.Combine(executableDir, "Plugins"); -Log.Information("Starting DiunaBI application"); -Log.Information("Loading plugins from: {PluginsPath}", pluginsPath); +if (app.Environment.IsProduction()) +{ + Log.Information("Starting DiunaBI application"); + Log.Information("Loading plugins from: {PluginsPath}", pluginsPath); +} +else +{ + var logger = app.Services.GetRequiredService>(); + logger.LogInformation("Starting DiunaBI application (Development)"); + logger.LogInformation("Loading plugins from: {PluginsPath}", pluginsPath); +} pluginManager.LoadPluginsFromDirectory(pluginsPath); @@ -143,5 +156,7 @@ app.MapControllers(); app.Run(); -// ✅ DODAJ CLEANUP -Log.CloseAndFlush(); +if (app.Environment.IsProduction()) +{ + Log.CloseAndFlush(); +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.sln b/src/Backend/DiunaBI.sln index 9fdd2df..7756af7 100644 --- a/src/Backend/DiunaBI.sln +++ b/src/Backend/DiunaBI.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 @@ -9,8 +9,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Core", "DiunaBI.Cor EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Database", "DiunaBI.Database\DiunaBI.Database.csproj", "{8C346BEA-A209-4E8F-A6BF-70B42D9106C8}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/src/Frontend/src/environments/environment.ts b/src/Frontend/src/environments/environment.ts index 98dcec0..b4ed8cc 100644 --- a/src/Frontend/src/environments/environment.ts +++ b/src/Frontend/src/environments/environment.ts @@ -7,8 +7,8 @@ export const environment = { appName: "LOCAL_DiunaBI", production: false, api: { - //url: "http://localhost:5400/api" - url: "https://diunabi-morska.bim-it.pl/api" + url: "http://localhost:5400/api" + //url: "https://diunabi-morska.bim-it.pl/api" }, google: { clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com" diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index 784b5ed..6b35da4 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5- +GET http://localhost:5400/api/Layers/RunQueueJobs/10763478CB738D4ecb2h76g803478CB738D4e diff --git a/tools/http-tests/AutoProcess.http b/tools/http-tests/AutoProcess.http index 97419fc..c807c75 100644 --- a/tools/http-tests/AutoProcess.http +++ b/tools/http-tests/AutoProcess.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoProcess/10763478CB738D4ecb2h76g803478CB738D4e +GET http://localhost:5400/api/Layers/CheckProcessors Timeout: 500000 diff --git a/tools/http-tests/BackupDatabase.http b/tools/http-tests/BackupDatabase.http index 539564b..043e64e 100644 --- a/tools/http-tests/BackupDatabase.http +++ b/tools/http-tests/BackupDatabase.http @@ -1,2 +1,2 @@ ### -GET http://localhost:5400/api/Admin/BackupDatabase/10763478CB738D4ecb2h76g803478CB738D4e +GET http://localhost:5400/api/Layers/AddPluginName