From b73adb4b171ecb0d25c1b11e2899eed27ccef95e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 20:23:36 +0200 Subject: [PATCH 01/11] Morska.Import.Standard refactored --- Utils/getProductionDatabase.sh | 2 +- Utils/totalCleanup.sh | 8 + src/Backend/DiunaBI.Core/Models/Record.cs | 2 +- .../DiunaBI.Core/Services/ProcessHelper.cs | 2 +- .../Importers/MorskaD1Importer.cs | 2 +- .../Importers/MorskaD3Importer.cs | 2 +- .../Importers/MorskaFK2Importer.cs | 2 +- .../Importers/MorskaImporter.cs | 240 +++++++++++++++--- .../Controllers/LayersController.cs | 154 +++++++++-- src/Backend/DiunaBI.WebAPI/Program.cs | 4 +- src/Frontend/src/environments/environment.ts | 4 +- tools/http-tests/AutoImport.http | 2 +- 12 files changed, 354 insertions(+), 70 deletions(-) create mode 100644 Utils/totalCleanup.sh diff --git a/Utils/getProductionDatabase.sh b/Utils/getProductionDatabase.sh index a0be17f..6e229e1 100644 --- a/Utils/getProductionDatabase.sh +++ b/Utils/getProductionDatabase.sh @@ -13,7 +13,7 @@ DOCKER_SA_PASSWORD="$&#ojoOOKEJ223" DOCKER_SQLCMD="/opt/mssql-tools18/bin/sqlcmd" DOCKER_BACKUP_PATH=${DOCKER_BACKUP_DIR}/${BACKUP_FILE} #SERVER VARIABLES -REMOTE_HOST="crm.bim-it.pl" +REMOTE_HOST="bim-it.pl" REMOTE_USER="mz" REMOTE_SA_PASSWORD="v](8Lc|RfG" REMOTE_BACKUP_DIR="/tmp" diff --git a/Utils/totalCleanup.sh b/Utils/totalCleanup.sh new file mode 100644 index 0000000..f77ab94 --- /dev/null +++ b/Utils/totalCleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +docker stop $(docker ps -aq) +docker rm -f $(docker ps -aq) +docker rmi -f $(docker images -q) +docker network prune -f +docker volume prune -f +docker builder prune -af \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Models/Record.cs b/src/Backend/DiunaBI.Core/Models/Record.cs index f791f06..a554c56 100644 --- a/src/Backend/DiunaBI.Core/Models/Record.cs +++ b/src/Backend/DiunaBI.Core/Models/Record.cs @@ -45,7 +45,7 @@ public class Record public double? Value32 { get; set; } //Description fields [StringLength(10000)] - public string? Desc1 { get; init; } + public string? Desc1 { get; set; } public DateTime CreatedAt { get; set; } public DateTime ModifiedAt { get; set; } public bool IsDeleted { get; init; } diff --git a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs index 46c7b0c..dcb3c8d 100644 --- a/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs +++ b/src/Backend/DiunaBI.Core/Services/ProcessHelper.cs @@ -4,6 +4,7 @@ using System.Text.RegularExpressions; using DiunaBI.Core.Models; namespace DiunaBI.Core.Services; + public static class ProcessHelper { public static void SetValue(Record record, int number, double? value) @@ -184,7 +185,6 @@ public static class ProcessHelper } return null; } - public static string GetSheetName(int month, int year) { if (month < 1 || month > 12) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 322cba4..d509374 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD1Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD1"; + public override string ImporterType => "Morska.Import.D1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 59f5afb..0050947 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaD3Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaD3"; + public override string ImporterType => "Morska.Import.D3"; private readonly AppDbContext _db; private readonly ILogger _logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 62c9de0..8b25067 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaFk2Importer : MorskaBaseImporter { - public override string ImporterType => "MorskaFK2"; + public override string ImporterType => "Morska.Import.FK2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs index b655f15..4f8ac22 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -8,12 +9,23 @@ namespace DiunaBI.Plugins.Morska.Importers; public class MorskaImporter : MorskaBaseImporter { - public override string ImporterType => "MorskaImporter"; - + public override string ImporterType => "Morska.Import.Standard"; + private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaImporter( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,47 +38,193 @@ public class MorskaImporter : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaImporter: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + // Load configuration from layer records + LoadConfiguration(importWorker); + + // Check if import should be performed + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual import + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + // Check date range if configured + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + // Outside date range - check if imported layer is up to date + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + // No date constraints - always import + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; + + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; // Assume up to date if no data + } + + // Check if the number of columns matches + if (data[0].Count != data[1].Count) + { + _logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; + } + + for (var i = 0; i < data[1].Count; i++) + { + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + { + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); + continue; + } + + // Check if the record exists in the database - add null check + var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); + if (existingRecord == null || existingRecord.Value1 != value) + { + _logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}", + ImporterType, data[0][i]); + return false; + } + } + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - _logger.LogDebug("MorskaImporter: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + _logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { @@ -79,25 +237,29 @@ public class MorskaImporter : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get( + SheetId!, + $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaImporter: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); if (data != null && data.Count >= 2) { for (var i = 0; i < data[1].Count; i++) { - if (!(data[0][i].ToString()?.Length > 0) || - !double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) + if (string.IsNullOrEmpty(data[0][i]?.ToString()) || + !double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) { - _logger.LogDebug("MorskaImporter: Skipping column {Index} - empty code or invalid value", i); + _logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value", + ImporterType, i); continue; } @@ -114,16 +276,16 @@ public class MorskaImporter : MorskaBaseImporter } _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaImporter: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaImporter: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } @@ -146,6 +308,12 @@ public class MorskaImporter : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaImporter: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 0a35f32..e26e973 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -20,15 +20,15 @@ public class LayersController : Controller private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; - private readonly ILogger _logger; + private readonly ILogger _logger; public LayersController( AppDbContext db, - SpreadsheetsResource.ValuesResource? googleSheetValues, + SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, - ILogger logger + ILogger logger ) { _db = db; @@ -59,7 +59,7 @@ public class LayersController : Controller .OrderByDescending(x => x.Number) .Skip(start).Take(limit).AsNoTracking().ToList(); - _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", + _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", result.Count, name, type); return Ok(result); @@ -228,7 +228,7 @@ public class LayersController : Controller _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); export.Export(layer); _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); - + return Ok(true); } catch (Exception e) @@ -276,7 +276,7 @@ public class LayersController : Controller } catch (Exception e) { - _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", + _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } @@ -328,7 +328,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", + _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", nameFilter, importWorkerLayers.Count); try @@ -345,8 +345,8 @@ public class LayersController : Controller { var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; - - _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", + + _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", importWorker.Name, type, source); if (source == "DataInbox" && type == "Import-D3") @@ -357,8 +357,8 @@ public class LayersController : Controller throw new Exception("MorskaD3 importer not found"); } d3Importer.Import(importWorker); - - _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", + + _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); continue; } @@ -374,7 +374,7 @@ public class LayersController : Controller d1importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -387,7 +387,7 @@ public class LayersController : Controller fk2importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); break; @@ -406,7 +406,7 @@ public class LayersController : Controller var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - + if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) { var importer = _pluginManager.GetImporter("MorskaImporter"); @@ -417,7 +417,7 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", + _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else if (IsImportedLayerUpToDate(importWorker) == false) @@ -430,12 +430,12 @@ public class LayersController : Controller importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", + _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } else { - _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", + _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", importWorker.Name, importWorker.Id); } break; @@ -443,7 +443,7 @@ public class LayersController : Controller } catch (Exception e) { - _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", + _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } } @@ -505,7 +505,7 @@ public class LayersController : Controller .AsNoTracking() .ToList(); - _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", + _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", type, processWorkerLayers.Count); foreach (var processWorker in processWorkerLayers) @@ -513,12 +513,12 @@ public class LayersController : Controller try { ProcessLayer(processWorker); - _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } catch (Exception e) { - _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", + _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", processWorker.Name, processWorker.Id, type); } } @@ -674,6 +674,112 @@ public class LayersController : Controller } } + [HttpGet] + [Route("GetImportWorkers")] + [AllowAnonymous] + public IActionResult GetImportWorkers() + { + var importWorkerLayers = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + && x.Number == 3487 + ) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + foreach (var importWorker in importWorkerLayers) + { + _logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; + if (pluginName != null) + { + var importer = _pluginManager.GetImporter(pluginName); + if (importer == null) + { + _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})", + pluginName, importWorker.Name, importWorker.Id); + throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}"); + } + try + { + _logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); + importer.Import(importWorker); + _logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + } + catch (Exception e) + { + _logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); + throw; + } + } + else + { + _logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + throw new Exception($"No plugin name found for import worker layer {importWorker.Name}"); + } + } + return Ok(); + } + + [HttpGet] + [Route("AddPluginName")] + [AllowAnonymous] + public IActionResult AddPluginName() + { + var record = new Record + { + Id = Guid.NewGuid(), + LayerId = Guid.Parse("eb5b4d0e-1607-4445-bbe5-65b9b8416787"), + Code = "Plugin", + Desc1 = "Morska.Import.Standard", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + _db.Records.Add(record); + _db.SaveChanges(); + + /* + var importWorkerLayers = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") + ) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + foreach (var importWorker in importWorkerLayers) + { + var record = new Record + { + Id = Guid.NewGuid(), + LayerId = importWorker.Id, + Code = "Plugin", + Desc1 = "Morska.Import.D3", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + //_db.Records.Add(record); + } + + //_db.SaveChanges(); + */ + return Ok(); + } + private static void WriteToConsole(params string[] messages) { @@ -734,7 +840,7 @@ public class LayersController : Controller var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (record == null) { - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", data[0][i].ToString(), importWorker.Name); isUpToDate = false; continue; @@ -752,12 +858,12 @@ public class LayersController : Controller continue; } - _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", record.Code, importWorker.Name); isUpToDate = false; } - _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", + _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", importWorker.Name, isUpToDate ? "up to date" : "outdated"); return isUpToDate; diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 0d62ca9..43dcced 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -125,7 +125,9 @@ app.Use(async (context, next) => if (token.Length > 0 && !context.Request.Path.ToString().Contains("getForPowerBI") && !context.Request.Path.ToString().Contains("getConfiguration") - && !context.Request.Path.ToString().Contains("DataInbox/Add")) + && !context.Request.Path.ToString().Contains("DataInbox/Add") + && !context.Request.Path.ToString().Contains("AddPluginName") // TODO: Remove this + && !context.Request.Path.ToString().Contains("GetImportWorkers")) { var handler = new JwtSecurityTokenHandler(); var data = handler.ReadJwtToken(token.Split(' ')[1]); diff --git a/src/Frontend/src/environments/environment.ts b/src/Frontend/src/environments/environment.ts index 98dcec0..b4ed8cc 100644 --- a/src/Frontend/src/environments/environment.ts +++ b/src/Frontend/src/environments/environment.ts @@ -7,8 +7,8 @@ export const environment = { appName: "LOCAL_DiunaBI", production: false, api: { - //url: "http://localhost:5400/api" - url: "https://diunabi-morska.bim-it.pl/api" + url: "http://localhost:5400/api" + //url: "https://diunabi-morska.bim-it.pl/api" }, google: { clientId: "107631825312-bkfe438ehr9k9ecb2h76g802tj6advma.apps.googleusercontent.com" diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index 784b5ed..53b13da 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoImport/10763478CB738D4ecb2h76g803478CB738D4e/K5- +GET http://localhost:5400/api/Layers/GetImportWorkers From ee9307c7c31db6f00435a997c1e3ec71f6562d90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 20:43:19 +0200 Subject: [PATCH 02/11] Morska.Import.D1 refactored --- .../Importers/MorskaD1Importer.cs | 297 ++++++++++++++---- .../Controllers/LayersController.cs | 2 +- 2 files changed, 243 insertions(+), 56 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index d509374..8e3b2af 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,17 @@ public class MorskaD1Importer : MorskaBaseImporter private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,51 +38,195 @@ public class MorskaD1Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD1: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaD1: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + if (data == null || data.Count < 2) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + foreach (var row in data) + { + if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue; + + var code = row[0].ToString(); + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + // Check values 3-17 (Value1-Value15) + for (int i = 3; i <= 17 && i < row.Count; i++) + { + var sheetValue = ParseValue(row[i]?.ToString()); + var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3 + + if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, i - 2, dbValue, sheetValue); + isUpToDate = false; + } + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { - Id = Guid.NewGuid(), + Id = Guid.NewGuid(), Number = _db.Layers.Count() + 1, ParentId = importWorker.Id, Type = LayerType.Import, @@ -79,17 +235,18 @@ public class MorskaD1Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaD1: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); var newRecords = (from t in data - where t.Count > 1 && (string)t[0] != string.Empty + where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString()) select new Record { Id = Guid.NewGuid(), @@ -114,20 +271,70 @@ public class MorskaD1Importer : MorskaBaseImporter }).ToList(); _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaD1: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD1: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } + private double? ParseValue(string? value) + { + if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; + value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); + try + { + double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); + return result; + } + catch (FormatException e) + { + _logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}", + ImporterType, value, e.Message); + return null; + } + } + + private bool IndexExists(IList array, int index) + { + return array != null && index >= 0 && index < array.Count; + } + + private double? GetRecordValueByIndex(Record record, int valueIndex) + { + return valueIndex switch + { + 1 => record.Value1, + 2 => record.Value2, + 3 => record.Value3, + 4 => record.Value4, + 5 => record.Value5, + 6 => record.Value6, + 7 => record.Value7, + 8 => record.Value8, + 9 => record.Value9, + 10 => record.Value10, + 11 => record.Value11, + 12 => record.Value12, + 13 => record.Value13, + 14 => record.Value14, + 15 => record.Value15, + _ => null + }; + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -146,27 +353,7 @@ public class MorskaD1Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD1: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); - } - - private double? ParseValue(string? value) - { - if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null; - value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray()); - try - { - double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result); - return result; - } - catch (FormatException e) - { - _logger.LogDebug("MorskaD1: Failed to parse value '{Value}': {Error}", value, e.Message); - return null; - } - } - - private bool IndexExists(IList array, int index) - { - return array != null && index >= 0 && index < array.Count; + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index e26e973..6fa9b27 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -684,7 +684,7 @@ public class LayersController : Controller .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 3487 + && x.Number == 8215 ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() From 5d4b25ab6b15e409c0d5811adfac430397e39b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 20:49:09 +0200 Subject: [PATCH 03/11] Morska.Import.D3 refactored --- .../Importers/MorskaD3Importer.cs | 206 +++++++++++++++--- 1 file changed, 173 insertions(+), 33 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index 0050947..f019bde 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.Json; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,15 @@ public class MorskaD3Importer : MorskaBaseImporter private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private string? ImportType { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaD3Importer( AppDbContext db, ILogger logger) @@ -24,67 +34,191 @@ public class MorskaD3Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaD3: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + try { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + ImportType = GetRecordValue(importWorker.Records, "ImportType"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + EndDate = endDate; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}", + ImporterType, importWorker.Name, ImportType); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1; - if (type == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportType not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - _logger.LogDebug("MorskaD3: Looking for DataInbox with type {Type}", type); + return true; + } - var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == type); + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; + } + + try + { + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); + if (dataInbox == null) + { + _logger.LogWarning("{ImporterType}: No DataInbox found for type {ImportType}", + ImporterType, ImportType); + return true; // Assume up to date if no data source + } + + // Compare timestamps - if DataInbox is newer than our layer, we need to import + var isUpToDate = newestLayer.CreatedAt >= dataInbox.CreatedAt; + + _logger.LogDebug("{ImporterType}: Layer created at {LayerTime}, DataInbox created at {DataTime}, up to date: {IsUpToDate}", + ImporterType, newestLayer.CreatedAt, dataInbox.CreatedAt, isUpToDate); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Looking for DataInbox with type {ImportType}", + ImporterType, ImportType); + + var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType); if (dataInbox == null) { - throw new Exception($"DataInbox not found, {type}"); + throw new InvalidOperationException($"DataInbox not found for type: {ImportType}"); } - _logger.LogDebug("MorskaD3: Found DataInbox {DataInboxId}, created at {CreatedAt}", - dataInbox.Id, dataInbox.CreatedAt); + _logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}", + ImporterType, dataInbox.Id, dataInbox.CreatedAt); try { var data = Convert.FromBase64String(dataInbox.Data); var jsonString = Encoding.UTF8.GetString(data); - _logger.LogDebug("MorskaD3: Decoded {DataSize} bytes from base64", data.Length); + _logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64", + ImporterType, data.Length); var records = JsonSerializer.Deserialize>(jsonString); if (records == null) { - throw new Exception($"DataInbox.Data is empty, {dataInbox.Name}"); + throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}"); } - _logger.LogDebug("MorskaD3: Deserialized {RecordCount} records from JSON", records.Count); + _logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON", + ImporterType, records.Count); - records = records.Where(x => x.Code!.StartsWith($"{year}{month}")).ToList(); + records = records.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList(); if (records.Count == 0) { - throw new Exception($"No records found for {year}{month}"); + throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}"); } - _logger.LogDebug("MorskaD3: Filtered to {FilteredCount} records for period {Year}{Month}", - records.Count, year, month); + _logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}", + ImporterType, records.Count, ImportYear, ImportMonth); records = records.Select(x => { @@ -105,23 +239,28 @@ public class MorskaD3Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; _db.Layers.Add(layer); - SaveRecords(layer.Id, records); _db.SaveChanges(); - _logger.LogInformation("MorskaD3: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - records.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, records.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaD3: Error processing DataInbox {DataInboxId}", dataInbox.Id); + _logger.LogError(e, "{ImporterType}: Error processing DataInbox {DataInboxId}", + ImporterType, dataInbox.Id); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -140,6 +279,7 @@ public class MorskaD3Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaD3: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file From edbb39c1cceb8334655df805d1359a6866a41c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 21:33:25 +0200 Subject: [PATCH 04/11] Morska.Import.FK2 refactored --- .../Importers/MorskaFK2Importer.cs | 253 +++++++++++++++--- .../Controllers/LayersController.cs | 2 +- 2 files changed, 218 insertions(+), 37 deletions(-) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index 8b25067..f94ce47 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; @@ -14,6 +15,17 @@ public class MorskaFk2Importer : MorskaBaseImporter private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties + private string? SheetId { get; set; } + private string? SheetTabName { get; set; } + private string? DataRange { get; set; } + private string? ImportYear { get; set; } + private string? ImportMonth { get; set; } + private string? ImportName { get; set; } + private DateTime? StartDate { get; set; } + private DateTime? EndDate { get; set; } + private bool IsEnabled { get; set; } + public MorskaFk2Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,47 +38,206 @@ public class MorskaFk2Importer : MorskaBaseImporter public override void Import(Layer importWorker) { - _logger.LogInformation("MorskaFK2: Starting import for {ImportWorkerName} ({ImportWorkerId})", - importWorker.Name, importWorker.Id); - - var sheetId = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetId")?.Desc1; - if (sheetId == null) + try { - throw new Exception($"SheetId not found, {importWorker.Name}"); + _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + + LoadConfiguration(importWorker); + + if (!ShouldPerformImport(importWorker)) + { + _logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return; + } + + ValidateConfiguration(); + + PerformImport(importWorker); + + _logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})", + ImporterType, importWorker.Name, importWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer importWorker) + { + if (importWorker.Records == null) return; + + SheetId = GetRecordValue(importWorker.Records, "SheetId"); + SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName"); + DataRange = GetRecordValue(importWorker.Records, "DataRange"); + ImportYear = GetRecordValue(importWorker.Records, "ImportYear"); + ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth"); + ImportName = GetRecordValue(importWorker.Records, "ImportName"); + IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True"; + + var startDateStr = GetRecordValue(importWorker.Records, "StartDate"); + if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate)) + { + StartDate = startDate; } - var sheetTabName = importWorker.Records!.FirstOrDefault(x => x.Code == "SheetTabName")?.Desc1; - if (sheetTabName == null) + var endDateStr = GetRecordValue(importWorker.Records, "EndDate"); + if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate)) { - throw new Exception($"SheetTabName not found, {importWorker.Name}"); + EndDate = endDate; } - var year = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportYear")?.Desc1; - if (year == null) + _logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}", + ImporterType, importWorker.Name); + } + + private bool ShouldPerformImport(Layer importWorker) + { + if (!IsEnabled) { - throw new Exception($"ImportYear not found, {importWorker.Name}"); + _logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var month = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportMonth")?.Desc1; - if (month == null) + if (StartDate.HasValue && EndDate.HasValue) { - throw new Exception($"ImportMonth not found, {importWorker.Name}"); + var now = DateTime.UtcNow.Date; + if (now >= StartDate.Value.Date && now <= EndDate.Value.Date) + { + _logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + if (!IsImportedLayerUpToDate(importWorker)) + { + _logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + _logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}", + ImporterType, importWorker.Name); + return false; } - var name = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportName")?.Desc1; - if (name == null) + return true; + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required"); + if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required"); + if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required"); + if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required"); + if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required"); + if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required"); + + if (errors.Any()) { - throw new Exception($"ImportName not found, {importWorker.Name}"); + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + } + + private bool IsImportedLayerUpToDate(Layer importWorker) + { + var newestLayer = _db.Layers + .Include(x => x.Records) + .Where(x => x.ParentId == importWorker.Id) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .FirstOrDefault(); + + if (newestLayer == null) + { + _logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date", + ImporterType, importWorker.Name); + return true; } - var dataRange = importWorker.Records!.FirstOrDefault(x => x.Code == "DataRange")?.Desc1; - if (dataRange == null) + try { - throw new Exception($"DataRange not found, {importWorker.Name}"); - } + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaFK2: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", - sheetId, sheetTabName, dataRange); + if (data == null || data.Count == 0) + { + _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", + ImporterType, importWorker.Name); + return true; + } + + var isUpToDate = true; + + for (var i = 0; i < data.Count; i++) + { + if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue; + + try + { + var dateArr = data[i][1].ToString()!.Split("."); + if (dateArr.Length != 3) continue; + + var number = data[i][2].ToString()!; + if (number.Length == 1) number = $"0{number}"; + var code = dateArr[2] + dateArr[1] + dateArr[0] + number; + + var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code); + if (record == null) + { + _logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}", + ImporterType, code, importWorker.Name); + isUpToDate = false; + continue; + } + + if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) && + Math.Abs((double)(record.Value1 - value)!) >= 0.01) + { + _logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}", + ImporterType, code, record.Value1, value); + isUpToDate = false; + } + + if (record.Desc1 != data[i][3]?.ToString()) + { + _logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}", + ImporterType, code, record.Desc1, data[i][3]?.ToString()); + isUpToDate = false; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison", + ImporterType, i); + isUpToDate = false; + } + } + + _logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}", + ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated"); + + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date", + ImporterType, importWorker.Name); + throw; + } + } + + private void PerformImport(Layer importWorker) + { + _logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}", + ImporterType, SheetId, SheetTabName, DataRange); var layer = new Layer { @@ -79,16 +250,17 @@ public class MorskaFk2Importer : MorskaBaseImporter CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - layer.Name = $"L{layer.Number}-I-{name}-{year}/{month}-{DateTime.Now:yyyyMMddHHmm}"; + layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}"; var newRecords = new List(); try { - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); var data = dataRangeResponse.Values; - _logger.LogDebug("MorskaFK2: Retrieved {RowCount} rows from Google Sheet", data?.Count ?? 0); + _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + ImporterType, data?.Count ?? 0); if (data != null) { @@ -96,15 +268,17 @@ public class MorskaFk2Importer : MorskaBaseImporter { if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - insufficient columns or empty desc", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc", + ImporterType, i); continue; } var dateArr = data[i][1].ToString()!.Split("."); if (dateArr.Length != 3) { - _logger.LogWarning("MorskaFK2: Invalid date format in row {Index}: {Date}", i, data[i][1]); - throw new Exception($"Invalid date in row {i}"); + _logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}", + ImporterType, i, data[i][1]); + throw new InvalidOperationException($"Invalid date in row {i}"); } var number = data[i][2].ToString()!; @@ -114,7 +288,8 @@ public class MorskaFk2Importer : MorskaBaseImporter if (string.IsNullOrEmpty(data[i][9]?.ToString()) || !double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value)) { - _logger.LogDebug("MorskaFK2: Skipping row {Index} - empty or invalid value", i); + _logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value", + ImporterType, i); continue; } @@ -132,20 +307,25 @@ public class MorskaFk2Importer : MorskaBaseImporter } _db.Layers.Add(layer); - SaveRecords(layer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("MorskaFK2: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", - newRecords.Count, layer.Name, layer.Id); + _logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})", + ImporterType, newRecords.Count, layer.Name, layer.Id); } catch (Exception e) { - _logger.LogError(e, "MorskaFK2: Error importing data from Google Sheet {SheetId}", sheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", + ImporterType, SheetId); throw; } } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } + private void SaveRecords(Guid layerId, ICollection records) { var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); @@ -164,6 +344,7 @@ public class MorskaFk2Importer : MorskaBaseImporter _db.Records.Add(record); } - _logger.LogDebug("MorskaFK2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}", + ImporterType, records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 6fa9b27..faed569 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -684,7 +684,7 @@ public class LayersController : Controller .Where(x => x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 8215 + && x.Number == 7270 ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() From e56be552741eb98973a387ff2b18dfbc9f4ac135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Fri, 6 Jun 2025 22:15:23 +0200 Subject: [PATCH 05/11] Move Databese stuff into Core and remove DiunaBI.Database Project --- .../Database}/Context/AppDbContext.cs | 2 +- .../Context/DesignTimeDbContextFactory.cs | 2 +- .../20221205190148_Initial.Designer.cs | 2 +- .../Migrations/20221205190148_Initial.cs | 0 ...1211210507_DataSetsAndDataRows.Designer.cs | 2 +- .../20221211210507_DataSetsAndDataRows.cs | 0 .../20221219163620_RenameFields.Designer.cs | 2 +- .../Migrations/20221219163620_RenameFields.cs | 0 ...21221165749_DataSetIdOnDataRow.Designer.cs | 2 +- .../20221221165749_DataSetIdOnDataRow.cs | 0 .../20230106095427_RenameModels.Designer.cs | 2 +- .../Migrations/20230106095427_RenameModels.cs | 0 .../20230626171614_LayerType.Designer.cs | 2 +- .../Migrations/20230626171614_LayerType.cs | 0 .../20230821105757_Record.Values.Designer.cs | 2 +- .../20230821105757_Record.Values.cs | 0 .../20230917110252_Layer.parent.Designer.cs | 2 +- .../Migrations/20230917110252_Layer.parent.cs | 0 .../20230918090621_ProcessSource.Designer.cs | 2 +- .../20230918090621_ProcessSource.cs | 0 .../20230918093055_TypeO.Designer.cs | 2 +- .../Migrations/20230918093055_TypeO.cs | 0 .../20231030142419_Record.Value32.Designer.cs | 2 +- .../20231030142419_Record.Value32.cs | 0 ...75645_Change record value type.Designer.cs | 2 +- ...20240309075645_Change record value type.cs | 0 ...240703171630_AfterCodeRefactor.Designer.cs | 2 +- .../20240703171630_AfterCodeRefactor.cs | 0 .../20240703173337_DataInboxModel.Designer.cs | 2 +- .../20240703173337_DataInboxModel.cs | 0 .../20240825144443_QueueJobs.Designer.cs | 2 +- .../Migrations/20240825144443_QueueJobs.cs | 0 .../20250317114722_LongerDesc1.Designer.cs | 2 +- .../Migrations/20250317114722_LongerDesc1.cs | 0 ...250529093632_LayersIsCancelled.Designer.cs | 2 +- .../20250529093632_LayersIsCancelled.cs | 0 .../Migrations/AppDbContextModelSnapshot.cs | 2 +- src/Backend/DiunaBI.Core/DiunaBI.Core.csproj | 16 +- .../Interfaces/IJobQueueService.cs | 18 ++ src/Backend/DiunaBI.Core/Models/QueueJob.cs | 71 ++++++-- .../DiunaBI.Core/Services/JobQueueService.cs | 155 ++++++++++++++++++ .../DiunaBI.Database/DiunaBI.Database.csproj | 27 --- .../DiunaBI.Plugins.Morska.csproj | 1 - .../Importers/MorskaD1Importer.cs | 2 +- .../Importers/MorskaD3Importer.cs | 2 +- .../Importers/MorskaFK2Importer.cs | 2 +- .../Importers/MorskaImporter.cs | 2 +- .../Processors/T1R1Processor.cs | 2 +- .../Processors/T1R3Processor.cs | 2 +- ...T3MultiSourceCopySelectedCodesProcessor.cs | 2 +- ...ceCopySelectedCodesYearSummaryProcessor.cs | 2 +- .../T3MultiSourceSummaryProcessor.cs | 2 +- .../T3MultiSourceYearSummaryProcessor.cs | 2 +- .../Processors/T3SingleSourceProcessor.cs | 2 +- .../T3SourceYearSummaryProcessor.cs | 2 +- .../Processors/T4R2Processor.cs | 2 +- .../Processors/T4SingleSourceProcessor.cs | 2 +- .../Processors/T5LastValuesProcessor.cs | 2 +- .../Controllers/AuthController.cs | 4 +- .../Controllers/DataInboxController.cs | 2 +- .../Controllers/LayersController.cs | 2 +- .../DiunaBI.WebAPI/DiunaBI.WebAPI.csproj | 1 - src/Backend/DiunaBI.WebAPI/Program.cs | 2 +- src/Backend/DiunaBI.sln | 4 +- 64 files changed, 285 insertions(+), 88 deletions(-) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Context/AppDbContext.cs (93%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Context/DesignTimeDbContextFactory.cs (96%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221205190148_Initial.Designer.cs (94%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221205190148_Initial.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs (96%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221211210507_DataSetsAndDataRows.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221219163620_RenameFields.Designer.cs (96%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221219163620_RenameFields.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs (96%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20221221165749_DataSetIdOnDataRow.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230106095427_RenameModels.Designer.cs (96%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230106095427_RenameModels.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230626171614_LayerType.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230626171614_LayerType.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230821105757_Record.Values.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230821105757_Record.Values.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230917110252_Layer.parent.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230917110252_Layer.parent.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230918090621_ProcessSource.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230918090621_ProcessSource.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230918093055_TypeO.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20230918093055_TypeO.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20231030142419_Record.Value32.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20231030142419_Record.Value32.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240309075645_Change record value type.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240309075645_Change record value type.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240703171630_AfterCodeRefactor.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240703171630_AfterCodeRefactor.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240703173337_DataInboxModel.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240703173337_DataInboxModel.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240825144443_QueueJobs.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20240825144443_QueueJobs.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20250317114722_LongerDesc1.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20250317114722_LongerDesc1.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20250529093632_LayersIsCancelled.Designer.cs (99%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/20250529093632_LayersIsCancelled.cs (100%) rename src/Backend/{DiunaBI.Database => DiunaBI.Core/Database}/Migrations/AppDbContextModelSnapshot.cs (99%) create mode 100644 src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs create mode 100644 src/Backend/DiunaBI.Core/Services/JobQueueService.cs delete mode 100644 src/Backend/DiunaBI.Database/DiunaBI.Database.csproj diff --git a/src/Backend/DiunaBI.Database/Context/AppDbContext.cs b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs similarity index 93% rename from src/Backend/DiunaBI.Database/Context/AppDbContext.cs rename to src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs index 3b5b0e0..ad56609 100644 --- a/src/Backend/DiunaBI.Database/Context/AppDbContext.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs @@ -2,7 +2,7 @@ using DiunaBI.Core.Models; using Microsoft.Extensions.Logging; -namespace DiunaBI.Database.Context; +namespace DiunaBI.Core.Database.Context; public class AppDbContext(DbContextOptions options) : DbContext(options) { diff --git a/src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs rename to src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs index 12add8d..378b43f 100644 --- a/src/Backend/DiunaBI.Database/Context/DesignTimeDbContextFactory.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs @@ -4,7 +4,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Configuration; -namespace DiunaBI.Database.Context; +namespace DiunaBI.Core.Database.Context; public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory { diff --git a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs similarity index 94% rename from src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs index e22e413..c0baa9b 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221205190148_Initial.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221205190148_Initial.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs index d1f6ba6..e5ac00e 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221211210507_DataSetsAndDataRows.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221211210507_DataSetsAndDataRows.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs index 3ca66f9..162b159 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221219163620_RenameFields.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221219163620_RenameFields.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs index e151324..1fdce1c 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20221221165749_DataSetIdOnDataRow.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20221221165749_DataSetIdOnDataRow.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs similarity index 96% rename from src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs index a48de5a..07be6ad 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230106095427_RenameModels.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230106095427_RenameModels.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs index 54ead09..517ba43 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230626171614_LayerType.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230626171614_LayerType.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs index 5787223..ae1da37 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230821105757_Record.Values.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230821105757_Record.Values.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs index e07e1d1..c7afb66 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230917110252_Layer.parent.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230917110252_Layer.parent.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs index 96cc281..37af610 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230918090621_ProcessSource.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918090621_ProcessSource.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs index fdc03a4..b19ab99 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.Designer.cs @@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20230918093055_TypeO.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20230918093055_TypeO.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs index b60890d..4b2411d 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20231030142419_Record.Value32.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20231030142419_Record.Value32.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs index c7faeaa..e226d0c 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240309075645_Change record value type.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240309075645_Change record value type.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs index fea2a0b..6ed6f16 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240703171630_AfterCodeRefactor.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703171630_AfterCodeRefactor.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs index 5b047dd..5ab24ba 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240703173337_DataInboxModel.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240703173337_DataInboxModel.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs index 54160aa..803a714 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20240825144443_QueueJobs.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20240825144443_QueueJobs.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs index 6fc4823..a50b999 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20250317114722_LongerDesc1.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250317114722_LongerDesc1.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs index 917717d..f2af26e 100644 --- a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.Designer.cs @@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.cs similarity index 100% rename from src/Backend/DiunaBI.Database/Migrations/20250529093632_LayersIsCancelled.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/20250529093632_LayersIsCancelled.cs diff --git a/src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs similarity index 99% rename from src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs rename to src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs index 0907c7e..3bc187d 100644 --- a/src/Backend/DiunaBI.Database/Migrations/AppDbContextModelSnapshot.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs @@ -5,7 +5,7 @@ using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; #nullable disable diff --git a/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj b/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj index 1920750..0a2988e 100644 --- a/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj +++ b/src/Backend/DiunaBI.Core/DiunaBI.Core.csproj @@ -9,7 +9,21 @@ - + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs b/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs new file mode 100644 index 0000000..25a70be --- /dev/null +++ b/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using DiunaBI.Core.Models; + +namespace DiunaBI.Core.Interfaces; + +public interface IJobQueueService +{ + Task EnqueueJobAsync(QueueJob job); + Task DequeueJobAsync(JobType? jobType = null); + Task MarkJobCompletedAsync(Guid jobId); + Task MarkJobFailedAsync(Guid jobId, string error); + Task MarkJobForRetryAsync(Guid jobId, string error); + Task> GetQueueStatusAsync(); + Task GetQueueCountAsync(JobType? jobType = null); + Task GetRunningJobsCountAsync(JobType jobType); +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Models/QueueJob.cs b/src/Backend/DiunaBI.Core/Models/QueueJob.cs index 54f5fdb..e7a4a2c 100644 --- a/src/Backend/DiunaBI.Core/Models/QueueJob.cs +++ b/src/Backend/DiunaBI.Core/Models/QueueJob.cs @@ -3,27 +3,68 @@ using System.ComponentModel.DataAnnotations; namespace DiunaBI.Core.Models; -public enum JobStatus +public class QueueJob { - New, - Failed, - Success + [Key] + public Guid Id { get; set; } = Guid.NewGuid(); + + [Required] + public Guid LayerId { get; set; } + + [Required] + [MaxLength(200)] + public string LayerName { get; set; } = string.Empty; + + [Required] + [MaxLength(100)] + public string PluginName { get; set; } = string.Empty; + + [Required] + public JobType JobType { get; set; } + + public int Priority { get; set; } = 0; // 0 = highest priority + + [Required] + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + + public int RetryCount { get; set; } = 0; + + public int MaxRetries { get; set; } = 5; + + [Required] + public JobStatus Status { get; set; } = JobStatus.Pending; + + [MaxLength(1000)] + public string? LastError { get; set; } + + public DateTime? LastAttemptAt { get; set; } + + public DateTime? CompletedAt { get; set; } + + [Required] + public Guid CreatedById { get; set; } + + [Required] + public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow; + + [Required] + public Guid ModifiedById { get; set; } + + [Required] + public DateTime ModifiedAtUtc { get; set; } = DateTime.UtcNow; } public enum JobType { - ImportWorker, - ProcessWorker + Import = 0, + Process = 1 } -public class QueueJob +public enum JobStatus { - [Key] public Guid Id { get; set; } - [Required] public Guid LayerId { get; set; } - [Required] public int Attempts { get; set; } - [Required] public JobStatus Status { get; set; } = JobStatus.New; - [Required] public JobType Type { get; set; } = JobType.ImportWorker; - public string Message { get; set; } = string.Empty; - [Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow; - [Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow; + Pending, + Running, + Completed, + Failed, + Retrying } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueService.cs b/src/Backend/DiunaBI.Core/Services/JobQueueService.cs new file mode 100644 index 0000000..c45201c --- /dev/null +++ b/src/Backend/DiunaBI.Core/Services/JobQueueService.cs @@ -0,0 +1,155 @@ + +using Microsoft.Extensions.Logging; +using DiunaBI.Core.Models; +using DiunaBI.Core.Interfaces; +using System.Threading.Tasks; +using DiunaBI.Core.Database.Context; +using System; +using System.Linq; +using Microsoft.EntityFrameworkCore; +using System.Collections.Generic; + +namespace DiunaBI.Core.Services; + +public class JobQueueService : IJobQueueService +{ + private readonly AppDbContext _db; + private readonly ILogger _logger; + + public JobQueueService(AppDbContext db, ILogger logger) + { + _db = db; + _logger = logger; + } + + public async Task EnqueueJobAsync(QueueJob job) + { + _db.QueueJobs.Add(job); + await _db.SaveChangesAsync(); + + _logger.LogInformation("JobQueue: Enqueued {JobType} job {JobId} for layer {LayerName} with plugin {PluginName} (priority {Priority})", + job.JobType, job.Id, job.LayerName, job.PluginName, job.Priority); + } + + public async Task DequeueJobAsync(JobType? jobType = null) + { + var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending); + + if (jobType.HasValue) + { + query = query.Where(x => x.JobType == jobType.Value); + } + + var job = await query + .OrderBy(x => x.JobType) // Importers first (0), then Processors (1) + .ThenBy(x => x.Priority) // Then by priority + .ThenBy(x => x.CreatedAt) // Then FIFO + .FirstOrDefaultAsync(); + + if (job != null) + { + job.Status = JobStatus.Running; + job.LastAttemptAt = DateTime.UtcNow; + job.ModifiedAtUtc = DateTime.UtcNow; + await _db.SaveChangesAsync(); + + _logger.LogDebug("JobQueue: Dequeued {JobType} job {JobId} for layer {LayerName} (priority {Priority})", + job.JobType, job.Id, job.LayerName, job.Priority); + } + + return job; + } + + public async Task MarkJobCompletedAsync(Guid jobId) + { + var job = await _db.QueueJobs.FindAsync(jobId); + if (job != null) + { + job.Status = JobStatus.Completed; + job.CompletedAt = DateTime.UtcNow; + job.ModifiedAtUtc = DateTime.UtcNow; + await _db.SaveChangesAsync(); + + _logger.LogInformation("JobQueue: {JobType} job {JobId} completed successfully for layer {LayerName}", + job.JobType, jobId, job.LayerName); + } + } + + public async Task MarkJobFailedAsync(Guid jobId, string error) + { + var job = await _db.QueueJobs.FindAsync(jobId); + if (job != null) + { + job.Status = JobStatus.Failed; + job.LastError = error; + job.ModifiedAtUtc = DateTime.UtcNow; + await _db.SaveChangesAsync(); + + _logger.LogError("JobQueue: {JobType} job {JobId} failed permanently for layer {LayerName}: {Error}", + job.JobType, jobId, job.LayerName, error); + } + } + + public async Task MarkJobForRetryAsync(Guid jobId, string error) + { + var job = await _db.QueueJobs.FindAsync(jobId); + if (job != null) + { + job.RetryCount++; + job.LastError = error; + job.ModifiedAtUtc = DateTime.UtcNow; + + if (job.RetryCount >= job.MaxRetries) + { + await MarkJobFailedAsync(jobId, $"Max retries ({job.MaxRetries}) exceeded. Last error: {error}"); + return; + } + + job.Status = JobStatus.Retrying; + await _db.SaveChangesAsync(); + + // Schedule retry with exponential backoff + var delayMinutes = Math.Pow(2, job.RetryCount); + _ = Task.Delay(TimeSpan.FromMinutes(delayMinutes)) + .ContinueWith(async _ => + { + var retryJob = await _db.QueueJobs.FindAsync(jobId); + if (retryJob?.Status == JobStatus.Retrying) + { + retryJob.Status = JobStatus.Pending; + retryJob.ModifiedAtUtc = DateTime.UtcNow; + await _db.SaveChangesAsync(); + + _logger.LogWarning("JobQueue: {JobType} job {JobId} re-queued for retry {RetryCount}/{MaxRetries} for layer {LayerName}", + retryJob.JobType, jobId, retryJob.RetryCount, retryJob.MaxRetries, retryJob.LayerName); + } + }); + } + } + + public async Task> GetQueueStatusAsync() + { + return await _db.QueueJobs + .OrderBy(x => x.JobType) + .ThenBy(x => x.Priority) + .ThenBy(x => x.CreatedAt) + .ToListAsync(); + } + + public async Task GetQueueCountAsync(JobType? jobType = null) + { + var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending); + + if (jobType.HasValue) + { + query = query.Where(x => x.JobType == jobType.Value); + } + + return await query.CountAsync(); + } + + public async Task GetRunningJobsCountAsync(JobType jobType) + { + return await _db.QueueJobs.CountAsync(x => x.Status == JobStatus.Running && x.JobType == jobType); + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj b/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj deleted file mode 100644 index e3d6e0b..0000000 --- a/src/Backend/DiunaBI.Database/DiunaBI.Database.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - net8.0 - enable - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj index f208118..a3345a5 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj +++ b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj @@ -12,6 +12,5 @@ - \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 8e3b2af..52d3975 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs index f019bde..ffae136 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs @@ -4,7 +4,7 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index f94ce47..d8dd909 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs index 4f8ac22..15cc887 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs @@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4; using Microsoft.Extensions.Logging; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index d9d87fa..aa66b0c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using DiunaBI.Core.Services.Calculations; using Microsoft.Extensions.Logging; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs index 6a6f633..7549f5a 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs @@ -5,7 +5,7 @@ using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs index c35f36c..277b0d4 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs @@ -1,7 +1,7 @@ using DiunaBI.Core.Services; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs index d54728b..92f5089 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs @@ -1,7 +1,7 @@ using DiunaBI.Core.Services; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index b4a7cb7..aebac1b 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -2,7 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services.Calculations; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index 90f7361..b418e7e 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -1,7 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using DiunaBI.Core.Services.Calculations; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs index a4c9c9d..138c89a 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using Microsoft.Extensions.Logging; using Google.Apis.Sheets.v4; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 85354e9..09b6972 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using Google.Apis.Sheets.v4; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index edb9e73..6a129f0 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -4,7 +4,7 @@ using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs index 1d44314..f58f1ca 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; using Google.Apis.Sheets.v4; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 5b3bc56..55dcf9b 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -1,7 +1,7 @@ using DiunaBI.Core.Services; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs index 7ae6717..3bb10b0 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/AuthController.cs @@ -6,8 +6,8 @@ using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; using System.Text; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; - +using DiunaBI.Core.Database.Context; + namespace DiunaBI.WebAPI.Controllers; [ApiController] diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs index 10ad60f..27cf379 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs @@ -3,7 +3,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Models; namespace DiunaBI.WebAPI.Controllers; diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index faed569..b80d80f 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -5,7 +5,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using Google.Cloud.Firestore; diff --git a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj index 6f45a65..6f4c3de 100644 --- a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj +++ b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj @@ -21,7 +21,6 @@ - diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 43dcced..071e540 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -6,7 +6,7 @@ using Microsoft.IdentityModel.Tokens; using System.IdentityModel.Tokens.Jwt; using System.Reflection; using System.Text; -using DiunaBI.Database.Context; +using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using Google.Apis.Sheets.v4; using Serilog; diff --git a/src/Backend/DiunaBI.sln b/src/Backend/DiunaBI.sln index 9fdd2df..7756af7 100644 --- a/src/Backend/DiunaBI.sln +++ b/src/Backend/DiunaBI.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 @@ -9,8 +9,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Core", "DiunaBI.Cor EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Database", "DiunaBI.Database\DiunaBI.Database.csproj", "{8C346BEA-A209-4E8F-A6BF-70B42D9106C8}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU From 787df8b838fff2e8ab612f1a13e226cf57def07b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 7 Jun 2025 12:33:33 +0200 Subject: [PATCH 06/11] JobQueue for import layers --- .../Database/Context/AppDbContext.cs | 10 - .../Context/DesignTimeDbContextFactory.cs | 6 +- ...0250607084540_QueueJobRefactor.Designer.cs | 415 ++++++++++++++++++ .../20250607084540_QueueJobRefactor.cs | 170 +++++++ .../Migrations/AppDbContextModelSnapshot.cs | 88 ++-- .../Services/JobQueueProcessor.cs | 202 +++++++++ .../Controllers/LayersController.cs | 110 +++-- src/Backend/DiunaBI.WebAPI/Program.cs | 85 ++-- 8 files changed, 972 insertions(+), 114 deletions(-) create mode 100644 src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs create mode 100644 src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs create mode 100644 src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs diff --git a/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs index ad56609..549d369 100644 --- a/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/AppDbContext.cs @@ -21,14 +21,4 @@ public class AppDbContext(DbContextOptions options) : DbContext(op x.SourceId }); } - - private static readonly LoggerFactory MyLoggerFactory = - new(new[] { - new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider() - }); - - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - optionsBuilder.UseLoggerFactory(MyLoggerFactory); - } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs index 378b43f..1da1ba1 100644 --- a/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs +++ b/src/Backend/DiunaBI.Core/Database/Context/DesignTimeDbContextFactory.cs @@ -11,17 +11,17 @@ public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory(); - var connectionString = configuration.GetConnectionString("DefaultConnection"); + var connectionString = configuration.GetConnectionString("SQLDatabase"); if (string.IsNullOrEmpty(connectionString)) { - throw new InvalidOperationException("Connection string 'DefaultConnection' not found in appsettings.json"); + throw new InvalidOperationException("Connection string 'SQLDatabase' not found in appsettings.json"); } optionsBuilder.UseSqlServer(connectionString); diff --git a/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs new file mode 100644 index 0000000..aec761a --- /dev/null +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs @@ -0,0 +1,415 @@ +// +using System; +using DiunaBI.Core.Database.Context; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DiunaBI.Core.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20250607084540_QueueJobRefactor")] + partial class QueueJobRefactor + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Data") + .IsRequired() + .HasMaxLength(2147483647) + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("DataInbox"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("IsCancelled") + .HasColumnType("bit"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Number") + .HasColumnType("int"); + + b.Property("ParentId") + .HasColumnType("uniqueidentifier"); + + b.Property("Type") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("ModifiedById"); + + b.HasIndex("ParentId"); + + b.ToTable("Layers"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => + { + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("SourceId") + .HasColumnType("uniqueidentifier"); + + b.HasKey("LayerId", "SourceId"); + + b.HasIndex("SourceId"); + + b.ToTable("ProcessSources"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CompletedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("JobType") + .HasColumnType("int"); + + b.Property("LastAttemptAt") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("LayerName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MaxRetries") + .HasColumnType("int"); + + b.Property("ModifiedAtUtc") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PluginName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Priority") + .HasColumnType("int"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.ToTable("QueueJobs"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Code") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Desc1") + .HasMaxLength(10000) + .HasColumnType("nvarchar(max)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("LayerId") + .HasColumnType("uniqueidentifier"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("Value1") + .HasColumnType("float"); + + b.Property("Value10") + .HasColumnType("float"); + + b.Property("Value11") + .HasColumnType("float"); + + b.Property("Value12") + .HasColumnType("float"); + + b.Property("Value13") + .HasColumnType("float"); + + b.Property("Value14") + .HasColumnType("float"); + + b.Property("Value15") + .HasColumnType("float"); + + b.Property("Value16") + .HasColumnType("float"); + + b.Property("Value17") + .HasColumnType("float"); + + b.Property("Value18") + .HasColumnType("float"); + + b.Property("Value19") + .HasColumnType("float"); + + b.Property("Value2") + .HasColumnType("float"); + + b.Property("Value20") + .HasColumnType("float"); + + b.Property("Value21") + .HasColumnType("float"); + + b.Property("Value22") + .HasColumnType("float"); + + b.Property("Value23") + .HasColumnType("float"); + + b.Property("Value24") + .HasColumnType("float"); + + b.Property("Value25") + .HasColumnType("float"); + + b.Property("Value26") + .HasColumnType("float"); + + b.Property("Value27") + .HasColumnType("float"); + + b.Property("Value28") + .HasColumnType("float"); + + b.Property("Value29") + .HasColumnType("float"); + + b.Property("Value3") + .HasColumnType("float"); + + b.Property("Value30") + .HasColumnType("float"); + + b.Property("Value31") + .HasColumnType("float"); + + b.Property("Value32") + .HasColumnType("float"); + + b.Property("Value4") + .HasColumnType("float"); + + b.Property("Value5") + .HasColumnType("float"); + + b.Property("Value6") + .HasColumnType("float"); + + b.Property("Value7") + .HasColumnType("float"); + + b.Property("Value8") + .HasColumnType("float"); + + b.Property("Value9") + .HasColumnType("float"); + + b.HasKey("Id"); + + b.HasIndex("CreatedById"); + + b.HasIndex("LayerId"); + + b.HasIndex("ModifiedById"); + + b.ToTable("Records"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Email") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("UserName") + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.HasKey("Id"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.Layer", "Parent") + .WithMany() + .HasForeignKey("ParentId"); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + + b.Navigation("Parent"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => + { + b.HasOne("DiunaBI.Core.Models.Layer", "Source") + .WithMany() + .HasForeignKey("SourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Source"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => + { + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") + .WithMany() + .HasForeignKey("CreatedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.Layer", null) + .WithMany("Records") + .HasForeignKey("LayerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") + .WithMany() + .HasForeignKey("ModifiedById") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("CreatedBy"); + + b.Navigation("ModifiedBy"); + }); + + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => + { + b.Navigation("Records"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs new file mode 100644 index 0000000..895d2f1 --- /dev/null +++ b/src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.cs @@ -0,0 +1,170 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DiunaBI.Core.Migrations +{ + /// + public partial class QueueJobRefactor : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "Message", + table: "QueueJobs"); + + migrationBuilder.RenameColumn( + name: "Type", + table: "QueueJobs", + newName: "RetryCount"); + + migrationBuilder.RenameColumn( + name: "ModifiedAt", + table: "QueueJobs", + newName: "ModifiedAtUtc"); + + migrationBuilder.RenameColumn( + name: "Attempts", + table: "QueueJobs", + newName: "Priority"); + + migrationBuilder.AddColumn( + name: "CompletedAt", + table: "QueueJobs", + type: "datetime2", + nullable: true); + + migrationBuilder.AddColumn( + name: "CreatedAtUtc", + table: "QueueJobs", + type: "datetime2", + nullable: false, + defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified)); + + migrationBuilder.AddColumn( + name: "CreatedById", + table: "QueueJobs", + type: "uniqueidentifier", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.AddColumn( + name: "JobType", + table: "QueueJobs", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "LastAttemptAt", + table: "QueueJobs", + type: "datetime2", + nullable: true); + + migrationBuilder.AddColumn( + name: "LastError", + table: "QueueJobs", + type: "nvarchar(1000)", + maxLength: 1000, + nullable: true); + + migrationBuilder.AddColumn( + name: "LayerName", + table: "QueueJobs", + type: "nvarchar(200)", + maxLength: 200, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "MaxRetries", + table: "QueueJobs", + type: "int", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "ModifiedById", + table: "QueueJobs", + type: "uniqueidentifier", + nullable: false, + defaultValue: new Guid("00000000-0000-0000-0000-000000000000")); + + migrationBuilder.AddColumn( + name: "PluginName", + table: "QueueJobs", + type: "nvarchar(100)", + maxLength: 100, + nullable: false, + defaultValue: ""); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CompletedAt", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "CreatedAtUtc", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "CreatedById", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "JobType", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LastAttemptAt", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LastError", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "LayerName", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "MaxRetries", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "ModifiedById", + table: "QueueJobs"); + + migrationBuilder.DropColumn( + name: "PluginName", + table: "QueueJobs"); + + migrationBuilder.RenameColumn( + name: "RetryCount", + table: "QueueJobs", + newName: "Type"); + + migrationBuilder.RenameColumn( + name: "Priority", + table: "QueueJobs", + newName: "Attempts"); + + migrationBuilder.RenameColumn( + name: "ModifiedAtUtc", + table: "QueueJobs", + newName: "ModifiedAt"); + + migrationBuilder.AddColumn( + name: "Message", + table: "QueueJobs", + type: "nvarchar(max)", + nullable: false, + defaultValue: ""); + } + } +} diff --git a/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs index 3bc187d..c325531 100644 --- a/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs +++ b/src/Backend/DiunaBI.Core/Database/Migrations/AppDbContextModelSnapshot.cs @@ -1,11 +1,10 @@ // using System; +using DiunaBI.Core.Database.Context; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using DiunaBI.Core.Models; -using DiunaBI.Core.Database.Context; #nullable disable @@ -18,12 +17,12 @@ namespace DiunaBI.Core.Migrations { #pragma warning disable 612, 618 modelBuilder - .HasAnnotation("ProductVersion", "9.0.0") + .HasAnnotation("ProductVersion", "8.0.0") .HasAnnotation("Relational:MaxIdentifierLength", 128); SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - modelBuilder.Entity("WebAPI.Models.DataInbox", b => + modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -52,7 +51,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("DataInbox"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -101,7 +100,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("Layers"); }); - modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => { b.Property("LayerId") .HasColumnType("uniqueidentifier"); @@ -116,32 +115,63 @@ namespace DiunaBI.Core.Migrations b.ToTable("ProcessSources"); }); - modelBuilder.Entity("WebAPI.Models.QueueJob", b => + modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("uniqueidentifier"); - b.Property("Attempts") - .HasColumnType("int"); + b.Property("CompletedAt") + .HasColumnType("datetime2"); b.Property("CreatedAt") .HasColumnType("datetime2"); + b.Property("CreatedAtUtc") + .HasColumnType("datetime2"); + + b.Property("CreatedById") + .HasColumnType("uniqueidentifier"); + + b.Property("JobType") + .HasColumnType("int"); + + b.Property("LastAttemptAt") + .HasColumnType("datetime2"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + b.Property("LayerId") .HasColumnType("uniqueidentifier"); - b.Property("Message") + b.Property("LayerName") .IsRequired() - .HasColumnType("nvarchar(max)"); + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); - b.Property("ModifiedAt") - .HasColumnType("datetime2"); - - b.Property("Status") + b.Property("MaxRetries") .HasColumnType("int"); - b.Property("Type") + b.Property("ModifiedAtUtc") + .HasColumnType("datetime2"); + + b.Property("ModifiedById") + .HasColumnType("uniqueidentifier"); + + b.Property("PluginName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); + + b.Property("Priority") + .HasColumnType("int"); + + b.Property("RetryCount") + .HasColumnType("int"); + + b.Property("Status") .HasColumnType("int"); b.HasKey("Id"); @@ -149,7 +179,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("QueueJobs"); }); - modelBuilder.Entity("WebAPI.Models.Record", b => + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -289,7 +319,7 @@ namespace DiunaBI.Core.Migrations b.ToTable("Records"); }); - modelBuilder.Entity("WebAPI.Models.User", b => + modelBuilder.Entity("DiunaBI.Core.Models.User", b => { b.Property("Id") .ValueGeneratedOnAdd() @@ -311,21 +341,21 @@ namespace DiunaBI.Core.Migrations b.ToTable("Users"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { - b.HasOne("WebAPI.Models.User", "CreatedBy") + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") .WithMany() .HasForeignKey("CreatedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.User", "ModifiedBy") + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") .WithMany() .HasForeignKey("ModifiedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.Layer", "Parent") + b.HasOne("DiunaBI.Core.Models.Layer", "Parent") .WithMany() .HasForeignKey("ParentId"); @@ -336,9 +366,9 @@ namespace DiunaBI.Core.Migrations b.Navigation("Parent"); }); - modelBuilder.Entity("WebAPI.Models.ProcessSource", b => + modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b => { - b.HasOne("WebAPI.Models.Layer", "Source") + b.HasOne("DiunaBI.Core.Models.Layer", "Source") .WithMany() .HasForeignKey("SourceId") .OnDelete(DeleteBehavior.Cascade) @@ -347,21 +377,21 @@ namespace DiunaBI.Core.Migrations b.Navigation("Source"); }); - modelBuilder.Entity("WebAPI.Models.Record", b => + modelBuilder.Entity("DiunaBI.Core.Models.Record", b => { - b.HasOne("WebAPI.Models.User", "CreatedBy") + b.HasOne("DiunaBI.Core.Models.User", "CreatedBy") .WithMany() .HasForeignKey("CreatedById") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.Layer", null) + b.HasOne("DiunaBI.Core.Models.Layer", null) .WithMany("Records") .HasForeignKey("LayerId") .OnDelete(DeleteBehavior.Cascade) .IsRequired(); - b.HasOne("WebAPI.Models.User", "ModifiedBy") + b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy") .WithMany() .HasForeignKey("ModifiedById") .OnDelete(DeleteBehavior.Cascade) @@ -372,7 +402,7 @@ namespace DiunaBI.Core.Migrations b.Navigation("ModifiedBy"); }); - modelBuilder.Entity("WebAPI.Models.Layer", b => + modelBuilder.Entity("DiunaBI.Core.Models.Layer", b => { b.Navigation("Records"); }); diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs new file mode 100644 index 0000000..66c7387 --- /dev/null +++ b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs @@ -0,0 +1,202 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; +using DiunaBI.Core.Models; +using DiunaBI.Core.Interfaces; +using DiunaBI.Core.Database.Context; +using System; +using System.Threading.Tasks; +using System.Threading; +using System.Net.Http; +using System.Linq; + +namespace DiunaBI.Core.Services; + +public class JobQueueProcessor : BackgroundService +{ + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public JobQueueProcessor( + IServiceProvider serviceProvider, + ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("JobQueueProcessor: Started"); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + using var scope = _serviceProvider.CreateScope(); + var queueService = scope.ServiceProvider.GetRequiredService(); + + // First process all imports (they run sequentially due to Google Sheets API limits) + await ProcessJobType(queueService, JobType.Import, maxConcurrency: 1, stoppingToken); + + // Then process processors (can run in parallel within same priority) + await ProcessJobType(queueService, JobType.Process, maxConcurrency: 3, stoppingToken); + + // Wait before next cycle + await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + } + catch (OperationCanceledException) + { + _logger.LogInformation("JobQueueProcessor: Cancellation requested"); + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "JobQueueProcessor: Unexpected error in queue processor"); + await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + } + } + + _logger.LogInformation("JobQueueProcessor: Stopped"); + } + + private async Task ProcessJobType(IJobQueueService queueService, JobType jobType, int maxConcurrency, CancellationToken cancellationToken) + { + var runningJobs = await queueService.GetRunningJobsCountAsync(jobType); + + // Don't start new jobs if we're at max concurrency + if (runningJobs >= maxConcurrency) + { + return; + } + + var job = await queueService.DequeueJobAsync(jobType); + if (job == null) + { + return; + } + + _logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})", + job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority); + + // Process job asynchronously to allow parallel processing of processors + _ = Task.Run(async () => + { + try + { + await ProcessJobAsync(job, cancellationToken); + + // Add delay between imports to respect Google Sheets API limits + if (job.JobType == JobType.Import) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "JobQueueProcessor: Error in background job processing for {JobType} job {JobId}", + job.JobType, job.Id); + } + }, cancellationToken); + } + + private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken) + { + try + { + using var scope = _serviceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var pluginManager = scope.ServiceProvider.GetRequiredService(); + var queueService = scope.ServiceProvider.GetRequiredService(); + + // Get the layer with records + var layer = await dbContext.Layers + .Include(x => x.Records) + .FirstOrDefaultAsync(x => x.Id == job.LayerId && !x.IsDeleted, cancellationToken); + + if (layer == null) + { + _logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId); + await queueService.MarkJobFailedAsync(job.Id, "Layer not found"); + return; + } + + // Process based on job type + switch (job.JobType) + { + case JobType.Import: + var importer = pluginManager.GetImporter(job.PluginName); + if (importer == null) + { + _logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName); + await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found"); + return; + } + + _logger.LogInformation("JobQueueProcessor: Executing import for layer {LayerName} with plugin {PluginName}", + layer.Name, job.PluginName); + + importer.Import(layer); + break; + + case JobType.Process: + var processor = pluginManager.GetProcessor(job.PluginName); + if (processor == null) + { + _logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName); + await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found"); + return; + } + + _logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with plugin {PluginName}", + layer.Name, job.PluginName); + + processor.Process(layer); + break; + + default: + throw new ArgumentOutOfRangeException(nameof(job.JobType), job.JobType, "Unknown job type"); + } + + await queueService.MarkJobCompletedAsync(job.Id); + + _logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}", + job.JobType, layer.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}", + job.JobType, job.Id, job.LayerName); + + using var scope = _serviceProvider.CreateScope(); + var queueService = scope.ServiceProvider.GetRequiredService(); + + // Check if it's a retriable error + if (IsRetriableError(ex)) + { + await queueService.MarkJobForRetryAsync(job.Id, ex.Message); + } + else + { + await queueService.MarkJobFailedAsync(job.Id, ex.Message); + } + } + } + + private static bool IsRetriableError(Exception ex) + { + var message = ex.Message.ToLowerInvariant(); + + var retriableErrors = new[] + { + "quota", "rate limit", "timeout", "service unavailable", + "internal server error", "bad gateway", "gateway timeout", + "network", "connection" + }; + + return retriableErrors.Any(error => message.Contains(error)) || + ex is HttpRequestException || + ex is TimeoutException; + } +} \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index b80d80f..5288bd7 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; -using Google.Cloud.Firestore; +using DiunaBI.Core.Interfaces; namespace DiunaBI.WebAPI.Controllers; @@ -21,6 +21,7 @@ public class LayersController : Controller private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; private readonly ILogger _logger; + private readonly IJobQueueService _queueService; public LayersController( AppDbContext db, @@ -28,7 +29,8 @@ public class LayersController : Controller GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, - ILogger logger + ILogger logger, + IJobQueueService queueService ) { _db = db; @@ -37,6 +39,7 @@ public class LayersController : Controller _configuration = configuration; _pluginManager = pluginManager; _logger = logger; + _queueService = queueService; } [HttpGet] @@ -677,56 +680,81 @@ public class LayersController : Controller [HttpGet] [Route("GetImportWorkers")] [AllowAnonymous] - public IActionResult GetImportWorkers() + public async Task GetImportWorkers() { - var importWorkerLayers = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 7270 - ) - .OrderByDescending(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - - foreach (var importWorker in importWorkerLayers) + try { - _logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})", - importWorker.Name, importWorker.Id); - var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; - if (pluginName != null) + var importWorkerLayers = await _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + && x.Number == 5579 + ) + .OrderBy(x => x.CreatedAt) + .AsNoTracking() + .ToListAsync(); + + _logger.LogInformation("GetImportWorkers: Found {LayerCount} import worker layers to queue", + importWorkerLayers.Count); + + int queuedCount = 0; + + foreach (var importWorker in importWorkerLayers) { + var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; + if (string.IsNullOrEmpty(pluginName)) + { + _logger.LogWarning("GetImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", + importWorker.Name, importWorker.Id); + continue; + } + + // Check if plugin exists var importer = _pluginManager.GetImporter(pluginName); if (importer == null) { - _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})", + _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping", pluginName, importWorker.Name, importWorker.Id); - throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}"); + continue; } - try + + var job = new QueueJob { - _logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}", - importWorker.Name, importWorker.Id, pluginName); - importer.Import(importWorker); - _logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})", - importWorker.Name, importWorker.Id); - } - catch (Exception e) - { - _logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}", - importWorker.Name, importWorker.Id, pluginName); - throw; - } - } - else - { - _logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})", - importWorker.Name, importWorker.Id); - throw new Exception($"No plugin name found for import worker layer {importWorker.Name}"); + LayerId = importWorker.Id, + LayerName = importWorker.Name ?? "Unknown", + PluginName = pluginName, + JobType = JobType.Import, + Priority = 0, // All imports have same priority + MaxRetries = 5, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + + await _queueService.EnqueueJobAsync(job); + queuedCount++; + + _logger.LogDebug("GetImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); } + + var totalQueueSize = await _queueService.GetQueueCountAsync(); + + _logger.LogInformation("GetImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}", + queuedCount, totalQueueSize); + + return Ok(new { + Message = $"Queued {queuedCount} import jobs", + QueuedJobs = queuedCount, + TotalQueueSize = totalQueueSize, + SkippedLayers = importWorkerLayers.Count - queuedCount + }); + } + catch (Exception e) + { + _logger.LogError(e, "GetImportWorkers: Error queuing import workers"); + return BadRequest(e.ToString()); } - return Ok(); } [HttpGet] diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 071e540..2f29ace 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -10,20 +10,24 @@ using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; using Google.Apis.Sheets.v4; using Serilog; +using DiunaBI.Core.Interfaces; var builder = WebApplication.CreateBuilder(args); -// ✅ DODAJ SERILOG CONFIGURATION -builder.Host.UseSerilog((context, configuration) => +// ✅ SERILOG TYLKO DLA PRODUKCJI +if (builder.Environment.IsProduction()) { - configuration - .ReadFrom.Configuration(context.Configuration) - .Enrich.FromLogContext() - .Enrich.WithProperty("Application", "DiunaBI") - .Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown") - .Enrich.WithEnvironmentName() - .Enrich.WithMachineName(); -}); + builder.Host.UseSerilog((context, configuration) => + { + configuration + .ReadFrom.Configuration(context.Configuration) + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", "DiunaBI") + .Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown") + .Enrich.WithEnvironmentName() + .Enrich.WithMachineName(); + }); +} var connectionString = builder.Configuration.GetConnectionString("SQLDatabase"); builder.Services.AddDbContext(x => @@ -50,7 +54,6 @@ builder.Services.AddCors(options => builder.Services.AddControllers(); - builder.Services.AddAuthentication(options => { options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme; @@ -66,10 +69,13 @@ builder.Services.AddAuthentication(options => ValidateIssuerSigningKey = true, IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!)) }; - }); builder.Services.AddAuthentication(); +// Queue services +builder.Services.AddScoped(); +builder.Services.AddHostedService(); + // Zarejestruj Google Sheets dependencies builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -90,32 +96,46 @@ builder.Services.AddSingleton(); var app = builder.Build(); -app.UseSerilogRequestLogging(options => +// ✅ SERILOG REQUEST LOGGING TYLKO DLA PRODUKCJI +if (app.Environment.IsProduction()) { - options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; - options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + app.UseSerilogRequestLogging(options => { - diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); - diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); - - var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); - if (!string.IsNullOrEmpty(userAgent)) + options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => { - diagnosticContext.Set("UserAgent", userAgent); - } - - diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); - diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); - }; -}); + diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); + diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); + + var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); + if (!string.IsNullOrEmpty(userAgent)) + { + diagnosticContext.Set("UserAgent", userAgent); + } + + diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); + diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); + }; + }); +} var pluginManager = app.Services.GetRequiredService(); var executablePath = Assembly.GetExecutingAssembly().Location; var executableDir = Path.GetDirectoryName(executablePath)!; var pluginsPath = Path.Combine(executableDir, "Plugins"); -Log.Information("Starting DiunaBI application"); -Log.Information("Loading plugins from: {PluginsPath}", pluginsPath); +// ✅ RÓŻNE LOGGERY W ZALEŻNOŚCI OD ŚRODOWISKA +if (app.Environment.IsProduction()) +{ + Log.Information("Starting DiunaBI application"); + Log.Information("Loading plugins from: {PluginsPath}", pluginsPath); +} +else +{ + var logger = app.Services.GetRequiredService>(); + logger.LogInformation("Starting DiunaBI application (Development)"); + logger.LogInformation("Loading plugins from: {PluginsPath}", pluginsPath); +} pluginManager.LoadPluginsFromDirectory(pluginsPath); @@ -145,5 +165,8 @@ app.MapControllers(); app.Run(); -// ✅ DODAJ CLEANUP -Log.CloseAndFlush(); +// ✅ SERILOG CLEANUP TYLKO DLA PRODUKCJI +if (app.Environment.IsProduction()) +{ + Log.CloseAndFlush(); +} From d05dc34e97fadf8deef3d1d767c8335a1c983ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 7 Jun 2025 13:51:27 +0200 Subject: [PATCH 07/11] Refactor processors --- .../Importers/MorskaD1Importer.cs | 61 +- .../Importers/MorskaFK2Importer.cs | 61 +- ...aImporter.cs => MorskaStandardImporter.cs} | 70 +- .../Processors/T1R1Processor.cs | 544 ++++++++++----- .../Processors/T1R3Processor.cs | 452 ++++++++++--- ...T3MultiSourceCopySelectedCodesProcessor.cs | 273 ++++++-- ...ceCopySelectedCodesYearSummaryProcessor.cs | 200 +++++- .../T3MultiSourceSummaryProcessor.cs | 290 ++++++-- .../T3MultiSourceYearSummaryProcessor.cs | 312 ++++++--- .../Processors/T3SingleSourceProcessor.cs | 297 ++++++-- .../T3SourceYearSummaryProcessor.cs | 213 +++++- .../Processors/T4R2Processor.cs | 634 ++++++++++++------ .../Processors/T4SingleSourceProcessor.cs | 224 ++++++- .../Processors/T5LastValuesProcessor.cs | 282 ++++++-- 14 files changed, 2996 insertions(+), 917 deletions(-) rename src/Backend/DiunaBI.Plugins.Morska/Importers/{MorskaImporter.cs => MorskaStandardImporter.cs} (86%) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 52d3975..aec4c02 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -26,6 +26,10 @@ public class MorskaD1Importer : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -43,6 +47,10 @@ public class MorskaD1Importer : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + LoadConfiguration(importWorker); if (!ShouldPerformImport(importWorker)) @@ -65,6 +73,46 @@ public class MorskaD1Importer : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -164,8 +212,8 @@ public class MorskaD1Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); if (data == null || data.Count < 2) { @@ -239,10 +287,10 @@ public class MorskaD1Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); var newRecords = (from t in data @@ -279,8 +327,7 @@ public class MorskaD1Importer : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index d8dd909..e4ccda7 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -26,6 +26,10 @@ public class MorskaFk2Importer : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaFk2Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -43,6 +47,10 @@ public class MorskaFk2Importer : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + LoadConfiguration(importWorker); if (!ShouldPerformImport(importWorker)) @@ -65,6 +73,46 @@ public class MorskaFk2Importer : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -164,8 +212,8 @@ public class MorskaFk2Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); if (data == null || data.Count == 0) { @@ -256,10 +304,10 @@ public class MorskaFk2Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); if (data != null) @@ -315,8 +363,7 @@ public class MorskaFk2Importer : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs similarity index 86% rename from src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs rename to src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs index 15cc887..49a9b5e 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs @@ -7,13 +7,13 @@ using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; -public class MorskaImporter : MorskaBaseImporter +public class MorskaStandardImporter : MorskaBaseImporter { public override string ImporterType => "Morska.Import.Standard"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties private string? SheetId { get; set; } @@ -26,10 +26,13 @@ public class MorskaImporter : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } - public MorskaImporter( + private IList>? _cachedSheetData; + private string? _cachedDataKey; + + public MorskaStandardImporter( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; @@ -43,6 +46,10 @@ public class MorskaImporter : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // Clear cache before import + _cachedSheetData = null; + _cachedDataKey = null; + // Load configuration from layer records LoadConfiguration(importWorker); @@ -69,6 +76,45 @@ public class MorskaImporter : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -171,14 +217,13 @@ public class MorskaImporter : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + var data = GetSheetData(); if (data == null || data.Count < 2) { _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", ImporterType, importWorker.Name); - return true; // Assume up to date if no data + return true; } // Check if the number of columns matches @@ -199,7 +244,6 @@ public class MorskaImporter : MorskaBaseImporter continue; } - // Check if the record exists in the database - add null check var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (existingRecord == null || existingRecord.Value1 != value) { @@ -243,12 +287,9 @@ public class MorskaImporter : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get( - SheetId!, - $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); if (data != null && data.Count >= 2) @@ -284,8 +325,7 @@ public class MorskaImporter : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index aa66b0c..d5801e6 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -18,6 +18,12 @@ public class T1R1Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + private string? GoogleSheetName { get; set; } + public T1R1Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,162 +33,321 @@ public class T1R1Processor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + // Load Google Sheet name + GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(GoogleSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for each month + var newRecords = ProcessAllMonths(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) - .ToList(); + return processedLayer; + } + private List ProcessAllMonths() + { var newRecords = new List(); for (var month = 1; month < 14; month++) { - if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))) + // Skip future months (except month 13 which is summary) + if (Year > DateTime.UtcNow.Year || + (Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)) { + _logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}", + ProcessorType, Year, month); continue; } - var records = new List(); - foreach (var source in sources) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3") - ).Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); + _logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}", + ProcessorType, month, Year); - if (dataSource == null) - { - throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found."); - } + var monthRecords = ProcessSingleMonth(month); + newRecords.AddRange(monthRecords); - var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault(); - if (codesRecord != null) - { - var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); - records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!)))); - } - else - { - records.AddRange(dataSource.Records!); - } - } - - if (dynamicCodes != null) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - records.Add(calc.CalculateT1(records)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - } - - newRecords.AddRange(records.Select(x => new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - } - )); + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}", + ProcessorType, monthRecords.Count, month); } - if (isNew) + return newRecords; + } + + private List ProcessSingleMonth(int month) + { + var records = new List(); + + // Collect records from all sources + foreach (var source in Sources!) + { + var sourceRecords = GetSourceRecords(source, month); + records.AddRange(sourceRecords); + } + + // Process dynamic codes (calculations) + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(records, month); + records.AddRange(calculatedRecords); + } + + // Create final records with month suffix + var monthRecords = records.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + return monthRecords; + } + + private List GetSourceRecords(Record source, int month) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found"); + } + + var sourceRecords = new List(); + + // Check if there are specific codes configured for this source + var codesRecord = Sources! + .Where(x => x.Code == $"Codes-{source.Desc1}") + .FirstOrDefault(); + + if (codesRecord != null) + { + var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); + sourceRecords.AddRange(dataSource.Records! + .Where(x => codes.Contains(int.Parse(x.Code!)))); + + _logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes", + ProcessorType, source.Desc1, codes.Count); + } + else + { + sourceRecords.AddRange(dataSource.Records!); + + _logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records", + ProcessorType, source.Desc1, dataSource.Records!.Count); + } + + return sourceRecords; + } + + private List ProcessDynamicCodes(List records, int month) + { + var calculatedRecords = new List(); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) + { + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}", + ProcessorType, dynamicCode.Id, month); + continue; + } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}", + ProcessorType, calc.Expression, dynamicCode.Id, month); + continue; + } + + var calculatedRecord = calc.CalculateT1(records); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}", + ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}", + ProcessorType, dynamicCode.Desc1, month); + } + } + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (sheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - UpdateReport(processedLayer.Id, sheetName); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -193,92 +358,139 @@ public class T1R1Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, string sheetName) + private void UpdateGoogleSheetReport(Guid sourceId) { - const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; - var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4"); - var response = request.Execute(); + try + { + _logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); - var r1 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); + const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; + + // Get processed layer data + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); - var codesRow = response.Values[0]; + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + // Get codes from sheet header + var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute(); + var codesRow = codesResponse.Values[0]; + + // Update monthly data (months 1-12) + UpdateMonthlyData(sheetId, codesRow, processedLayer); + + // Update summary row (month 13) + UpdateSummaryData(sheetId, codesRow, processedLayer); + + // Update timestamps + UpdateTimestamps(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + throw; + } + } + + private void UpdateMonthlyData(string sheetId, IList codesRow, Layer processedLayer) + { var valueRange = new ValueRange { Values = new List>() }; - for (var i = 1; i <= 12; i++) + // Process months 1-12 + for (var month = 1; month <= 12; month++) { - var values = new List(); + var monthValues = new List(); foreach (string code in codesRow) { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}"); + monthValues.Add(record?.Value1?.ToString() ?? "0"); } - valueRange.Values.Add(values); + valueRange.Values.Add(monthValues); } - // sum - var valuesSum = new List(); - var emptyRow = new List(); - foreach (string code in codesRow) - { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); - emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } - } - - valueRange.Values.Add(emptyRow); - valueRange.Values.Add(valuesSum); - - var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20"); - update.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); - // update time - var timeUtc = new List - { - r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange + _logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType); + } + + private void UpdateSummaryData(string sheetId, IList codesRow, Layer processedLayer) + { + var valueRange = new ValueRange { Values = new List>() }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); + // Empty row + var emptyRow = new List(); + for (int i = 0; i < codesRow.Count; i++) + { + emptyRow.Add(""); + } + valueRange.Values.Add(emptyRow); + + // Summary row (month 13) + var summaryValues = new List(); + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + summaryValues.Add(record?.Value1?.ToString() ?? "0"); + } + valueRange.Values.Add(summaryValues); + + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs index 7549f5a..7a4cc34 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs @@ -18,6 +18,10 @@ public class T1R3Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + public T1R3Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,46 +31,140 @@ public class T1R3Processor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1; - if (source == null) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - - var newRecords = new List(); - - string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$"; + private List GetDataSources() + { + string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$"; + var dataSources = _db.Layers .Where(x => !x.IsDeleted && !x.IsCancelled) .Include(layer => layer.Records!) @@ -75,59 +173,127 @@ public class T1R3Processor : MorskaBaseProcessor .Where(x => Regex.IsMatch(x.Name!, pattern)) .ToList(); + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for pattern: {pattern}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}", + ProcessorType, dataSources.Count, pattern); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); + foreach (var dataSource in dataSources) { - var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); - if (month == null) + var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); + if (monthStr == null || !int.TryParse(monthStr, out var month)) { - throw new Exception($"Month not found: {dataSource.Name}"); + _logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}", + ProcessorType, dataSource.Name); + continue; } - foreach (var record in dataSource.Records!) + _logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}", + ProcessorType, dataSource.Name, month); + + var sourceRecords = ProcessDataSourceRecords(dataSource, month); + newRecords.AddRange(sourceRecords); + + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}", + ProcessorType, sourceRecords.Count, dataSource.Name); + } + + _logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDataSourceRecords(Layer dataSource, int month) + { + var newRecords = new List(); + + foreach (var record in dataSource.Records!) + { + if (record.Value1 == null) { - if (record.Value1 == null) continue; - for (var i = 1; i < 33; i++) + _logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null", + ProcessorType, record.Code); + continue; + } + + // Process values for positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(record, i); + if (value == null) { - if (ProcessHelper.GetValue(record, i) == null) continue; - - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{record.Code}{month}{i:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = i == 1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100, - Desc1 = record.Desc1 - }; - - newRecords.Add(newRecord); + continue; } + + var baseValue = (double)record.Value1!; + var positionValue = (double)value; + var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100; + + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"{record.Code}{month:D2}{i:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = calculatedValue, + Desc1 = record.Desc1 + }; + + newRecords.Add(newRecord); + + _logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}", + ProcessorType, newRecord.Code, newRecord.Value1, record.Code); } } - if (isNew) + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - UpdateReport(processedLayer.Id, year); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -138,90 +304,164 @@ public class T1R3Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, int year) + private void UpdateGoogleSheetReport(Guid sourceId) { - const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; - - var r3 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - for (var i = 1; i <= 12; i++) + try { - var sheetName = ProcessHelper.GetSheetName(i, year); + _logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}", + ProcessorType, sourceId); + + const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; + + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + // Update sheets for all months + for (var month = 1; month <= 12; month++) + { + UpdateMonthSheet(sheetId, processedLayer, month); + } + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month) + { + var sheetName = ProcessHelper.GetSheetName(month, Year); + + try + { + _logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + + // Get codes from sheet ValueRange? dataRangeResponse; try { dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); } - catch + catch (Exception e) { - continue; // Sheet not exist + _logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}", + ProcessorType, sheetName, e.Message); + return; } - if (dataRangeResponse == null) continue; // Sheet not exist - var data = dataRangeResponse.Values; - - var updateValueRange = new ValueRange + if (dataRangeResponse?.Values == null) { - Values = new List>() - }; - - foreach (var row in data) - { - if (row.Count == 0) continue; - var code = row[0].ToString(); - - var updateRow = new List(); - - for (var j = 1; j < 16; j++) - { - var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}"); - if (codeRecord is { Value1: not null }) - { - updateRow.Add(codeRecord.Value1); - } - else - { - updateRow.Add(""); - } - } - - updateValueRange.Values.Add(updateRow); + _logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping", + ProcessorType, sheetName); + return; } - dataRangeResponse.Values = data; - var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); - update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - update.Execute(); + // Update data + UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month); - // update time - var timeUtc = new List - { - r3!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r3.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange - { - Values = new List>() - }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); + // Update timestamps + UpdateSheetTimestamps(sheetId, sheetName, processedLayer); - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); + _logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}", + ProcessorType, sheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + throw; } } + + private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList> codeRows, int month) + { + var updateValueRange = new ValueRange + { + Values = new List>() + }; + + foreach (var row in codeRows) + { + if (row.Count == 0) continue; + + var code = row[0].ToString(); + var updateRow = new List(); + + // Process columns C to Q (positions 1-15) + for (var position = 1; position <= 15; position++) + { + var recordCode = $"{code}{month:D2}{position:D2}"; + var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode); + + if (codeRecord?.Value1 != null) + { + updateRow.Add(codeRecord.Value1); + _logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}", + ProcessorType, codeRecord.Value1, recordCode); + } + else + { + updateRow.Add(""); + } + } + + updateValueRange.Values.Add(updateRow); + } + + // Update sheet with new values + var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}", + ProcessorType, updateValueRange.Values.Count, sheetName); + } + + private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, sheetName, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs index 277b0d4..11fce8d 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs @@ -13,6 +13,13 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private string? Codes { get; set; } + private List? CodesList { get; set; } + public T3MultiSourceCopySelectedCodesProcessor( AppDbContext db, ILogger logger) @@ -20,102 +27,268 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); } - var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; - if (codes == null) + catch (Exception e) { - throw new Exception("Codes record not found"); + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var codesList = ProcessHelper.ParseCodes(codes); + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load codes + Codes = GetRecordValue(processWorker.Records, "Codes"); + if (string.IsNullOrEmpty(Codes)) + { + throw new InvalidOperationException("Codes record not found"); + } + + // Parse codes list + CodesList = ProcessHelper.ParseCodes(Codes); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}", + ProcessorType, Year, Month, Sources.Count, CodesList.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = new List(); + + foreach (var source in Sources!) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + _logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3", + ProcessorType, Year, Month, source.Desc1); + continue; + } + + dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + } - var dataSources = sources.Select(source => _db.Layers - .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) - .Include(x => x.Records).AsNoTracking() - .FirstOrDefault()) - .OfType() - .ToList(); if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources", + ProcessorType, dataSources.Count); - var newRecords = dataSources - .SelectMany(x => x.Records!) - .Where(x => codesList.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); - } - return newRecord; - }) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + var filteredRecords = allSourceRecords + .Where(x => !string.IsNullOrEmpty(x.Code) && + int.TryParse(x.Code, out var code) && + CodesList!.Contains(code)) .ToList(); - if (isNew) + + _logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes", + ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count); + + var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateCopiedRecord(Record sourceRecord) + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = sourceRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(sourceRecord, i); + ProcessHelper.SetValue(newRecord, i, value); + } + + _logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32", + ProcessorType, newRecord.Code); + + return newRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -126,6 +299,12 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs index 92f5089..dd7f50c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs @@ -13,6 +13,9 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + public T3MultiSourceCopySelectedCodesYearSummaryProcessor( AppDbContext db, ILogger logger) @@ -20,67 +23,176 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}", + ProcessorType, Year); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}", + ProcessorType, Year); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = new List(); - for (var i = 1; i < 13; i++) + for (var month = 1; month <= 12; month++) { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-AB-T3")) + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + if (dataSource != null) { dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); } } if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}"); } - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); - foreach (var baseRecord in dataSources.Last().Records!) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -88,34 +200,60 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - for (var i = 1; i < 33; i++) + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0); + ProcessHelper.SetValue(processedRecord, position, totalValue); } newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -126,6 +264,12 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index aebac1b..e7f642d 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -14,6 +14,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + public T3MultiSourceSummaryProcessor( AppDbContext db, ILogger logger) @@ -24,47 +30,159 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) @@ -73,17 +191,28 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -91,82 +220,105 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; + + // Sum values from all sources for positions 1-32 for (var i = 1; i < 33; i++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); } - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) { - foreach (var dynamicCode in dynamicCodes) + try { - try + if (string.IsNullOrEmpty(dynamicCode.Desc1)) { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; } - catch (Exception e) + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) { - _logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); } } - if (isNew) + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -177,6 +329,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index b418e7e..75b8f43 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -14,6 +14,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + public T3MultiSourceYearSummaryProcessor( AppDbContext db, ILogger logger) @@ -24,69 +29,180 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes with validation) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")) + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) .OfType() .ToList(); - + if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}"); } - var allRecords = dataSources - .SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); - var codeRecordsValidation = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -94,95 +210,119 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; + + // Validation record for double-checking calculations var validationRecord = new Record(); + + // Sum values from all sources for positions 1-32 with validation for (var i = 1; i < 33; i++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); - ProcessHelper.SetValue(validationRecord, i, - codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); + // Validation calculation (identical to main calculation) + var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(validationRecord, i, validationValue); - if ( - double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - - ProcessHelper.GetValue(validationRecord, i))!) > 0.01) + // Validate that both calculations match + var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!); + if (difference > 0.01) { - throw new Exception($"ValidationError: Code {baseCode}, " + - $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + - $"{ProcessHelper.GetValue(validationRecord, i)})"); + throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})"); } } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); } - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; + } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); } } - if (isNew) + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } - + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -193,6 +333,12 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs index 138c89a..57c0fa6 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs @@ -15,6 +15,13 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayerName { get; set; } + private string? Source { get; set; } + private Layer? SourceImportWorker { get; set; } + public T3SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -24,70 +31,184 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) + try { - throw new Exception("SourceLayer record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); } - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) + catch (Exception e) { - throw new Exception("SourceImportWorkerL layer not found"); + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; } - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) { - throw new Exception("Source record not found"); + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year and month + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer name + SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayerName)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source name + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayerName, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + // Find source import worker + SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled); + if (SourceImportWorker == null) + { + errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found"); + } + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = _db.Layers .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled) .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToList(); + if (dataSources.Count == 0) { - throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); + throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records", + ProcessorType, allRecords.Count); + foreach (var baseRecord in dataSources.Last().Records!) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); @@ -100,67 +221,104 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor ModifiedAt = DateTime.UtcNow }; - var lastDayInMonth = DateTime.DaysInMonth(year, month); - //day 1 - var firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0; - ProcessHelper.SetValue(processedRecord, 1, firstVal); - var previousValue = firstVal; - //days 2-29/30 - for (var i = 2; i < lastDayInMonth; i++) - { - var dayVal = codeRecords - .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1; - if (dayVal == null) - { - ProcessHelper.SetValue(processedRecord, i, 0); - } - else - { - var processedVal = dayVal - previousValue; - ProcessHelper.SetValue(processedRecord, i, processedVal); - previousValue = (double)dayVal; - } - } - //last day - var lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1; - - if (lastVal == null) - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); - } - else - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); - } - - // copy last value - var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; - ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + ProcessDailyValues(processedRecord, codeRecords); newRecords.Add(processedRecord); } + _logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void ProcessDailyValues(Record processedRecord, List codeRecords) + { + var lastDayInMonth = DateTime.DaysInMonth(Year, Month); + + // Day 1 - first value for the month + var firstVal = codeRecords + .Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1)) + .MaxBy(x => x.CreatedAt)?.Value1 ?? 0; + + ProcessHelper.SetValue(processedRecord, 1, firstVal); + var previousValue = firstVal; + + // Days 2 to last-1 - daily differences + for (var i = 2; i < lastDayInMonth; i++) + { + var dayVal = codeRecords + .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (dayVal == null) + { + ProcessHelper.SetValue(processedRecord, i, 0); + } + else + { + var processedVal = dayVal - previousValue; + ProcessHelper.SetValue(processedRecord, i, processedVal); + previousValue = (double)dayVal; + } + } + + // Last day - special handling + var lastVal = codeRecords + .Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth)) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (lastVal == null) + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); + } + else + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); + } + + // Copy last value to position 32 + var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; + ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + + _logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}", + ProcessorType, processedRecord.Code, valueToCopy); + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id); + if (isNew) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -171,7 +329,12 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 09b6972..0fcd769 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -15,6 +15,10 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + public T3SourceYearSummaryProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,72 +31,181 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = new List(); - for (var i = 1; i < 13; i++) + + for (var month = 1; month <= 12; month++) { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-{source}-T3")) + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + if (dataSource != null) { dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); } } if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}"); } - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}", + ProcessorType, dataSources.Count, Year, Source); - foreach (var baseRecord in dataSources.Last().Records!) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -100,38 +213,60 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - for (var i = 1; i < 33; i++) + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position)); + ProcessHelper.SetValue(processedRecord, position, totalValue); } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } - + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -142,6 +277,12 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index 6a129f0..e2fccf9 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -7,7 +7,6 @@ using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; - namespace DiunaBI.Plugins.Morska.Processors; public class T4R2Processor : MorskaBaseProcessor @@ -18,6 +17,13 @@ public class T4R2Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private string? LayerName { get; set; } + private string? ReportSheetName { get; set; } + private string? InvoicesSheetName { get; set; } + public T4R2Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -30,200 +36,365 @@ public class T4R2Processor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; - if (layerName == null) + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { - throw new Exception("LayerName record not found"); + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); } + // Load layer name + LayerName = GetRecordValue(processWorker.Records, "LayerName"); + if (string.IsNullOrEmpty(LayerName)) + { + throw new InvalidOperationException("LayerName record not found"); + } + + // Load report sheet name + ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(ReportSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + // Load invoices sheet name + InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices"); + if (string.IsNullOrEmpty(InvoicesSheetName)) + { + throw new InvalidOperationException("GoogleSheetName-Invoices record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}", + ProcessorType, Year, Sources.Count, LayerName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required"); + if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required"); + if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for all sources + var newRecords = ProcessSources(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheets reports + UpdateReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-{LayerName}"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - + return processedLayer; + } + private List ProcessSources() + { var newRecords = new List(); - foreach (var source in sources) + foreach (var source in Sources!) { - var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") - ?.Desc1; - var sourceCodes = new List(); - if (rawSourceCodes != null) - { - sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); - } + _logger.LogDebug("{ProcessorType}: Processing source {Source}", + ProcessorType, source.Desc1); - List lastSourceCodes = []; + var sourceCodes = GetSourceCodes(source); + var sourceRecords = ProcessSourceData(source, sourceCodes); + newRecords.AddRange(sourceRecords); - for (var month = 1; month <= 12; month++) - { - if ((year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || year < DateTime.UtcNow.Year) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSource != null) - { - lastSourceCodes = dataSource.Records!.Select(x => x.Code!).ToList(); - var news = dataSource.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, - Desc1 = x.Desc1 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id); - } - } - else - { - //0 values for future months - if (source.Desc1 == "FK2" || lastSourceCodes.Count <= 0) continue; - var news = lastSourceCodes - .Where(x => sourceCodes.Contains(int.Parse(x))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = 0, - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - } + _logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records", + ProcessorType, source.Desc1, sourceRecords.Count); + } - // year summary - var dataSourceSum = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSourceSum != null) + _logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List GetSourceCodes(Record source) + { + var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}"); + if (string.IsNullOrEmpty(rawSourceCodes)) + { + return new List(); + } + + return ProcessHelper.ParseCodes(rawSourceCodes); + } + + private List ProcessSourceData(Record source, List sourceCodes) + { + var sourceRecords = new List(); + var lastSourceCodes = new List(); + + // Process monthly data (1-12) + for (var month = 1; month <= 12; month++) + { + var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes); + sourceRecords.AddRange(monthRecords); + } + + // Process year summary (month 13) + var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes); + sourceRecords.AddRange(yearSummaryRecords); + + return sourceRecords; + } + + private List ProcessMonthData(Record source, List sourceCodes, int month, List lastSourceCodes) + { + var monthRecords = new List(); + + if (IsDataAvailableForMonth(month)) + { + var dataSource = GetMonthDataSource(source, month); + if (dataSource != null) { - var news = dataSourceSum.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}13", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); + lastSourceCodes.Clear(); + lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!)); + + var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes); + var processedRecords = CreateMonthRecords(filteredRecords, source, month); + monthRecords.AddRange(processedRecords); + + _logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records", + ProcessorType, month, source.Desc1, processedRecords.Count); } else { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Data source {DataSource} not found", + ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T"); + } + } + else + { + // Future months - create zero value records (except for FK2) + if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0) + { + var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month); + monthRecords.AddRange(futureRecords); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}", + ProcessorType, futureRecords.Count, month); } } - if (isNew) + return monthRecords; + } + + private bool IsDataAvailableForMonth(int month) + { + return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year; + } + + private Layer? GetMonthDataSource(Record source, int month) + { + return _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + } + + private List FilterRecords(IEnumerable records, List sourceCodes) + { + return records + .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) + .ToList(); + } + + private List CreateMonthRecords(List filteredRecords, Record source, int month) + { + return filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, + Desc1 = x.Desc1 + }).ToList(); + } + + private List CreateFutureMonthRecords(List lastSourceCodes, List sourceCodes, int month) + { + return lastSourceCodes + .Where(x => sourceCodes.Contains(int.Parse(x))) + .Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = 0 + }).ToList(); + } + + private List ProcessYearSummaryData(Record source, List sourceCodes) + { + var dataSourceSum = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSourceSum == null) + { + _logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found", + ProcessorType, $"{Year}/13-{source.Desc1}-T3"); + return new List(); + } + + var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes); + var yearSummaryRecords = filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}13", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}", + ProcessorType, yearSummaryRecords.Count, source.Desc1); + + return yearSummaryRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (reportSheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - var invoicesSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName-Invoices")?.Desc1; - if (invoicesSheetName == null) - { - throw new Exception("GoogleSheetName-Invoices record not found"); - } - - UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName); - - _logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -234,153 +405,196 @@ public class T4R2Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName) + private void UpdateReport(Guid sourceId) { - const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; - var request = _googleSheetValues.Get(sheetId, "C4:Z4"); - var response = request.Execute(); + try + { + _logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}", + ProcessorType, sourceId); - var r2 = _db.Layers + const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; + + var processedLayer = GetProcessedLayerData(sourceId); + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + var codesRow = GetCodesFromSheet(sheetId); + + UpdateMonthlyData(sheetId, processedLayer, codesRow); + UpdateYearSummary(sheetId, processedLayer, codesRow); + UpdateTimestamps(sheetId, processedLayer); + UpdateInvoicesData(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private Layer? GetProcessedLayerData(Guid sourceId) + { + return _db.Layers .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + } + private IList GetCodesFromSheet(string sheetId) + { + var request = _googleSheetValues.Get(sheetId, "C4:Z4"); + var response = request.Execute(); + return response.Values[0]; + } + + private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList codesRow) + { const int startRow = 6; - var codesRow = response.Values[0]; - for (var i = 1; i <= 12; i++) + for (var month = 1; month <= 12; month++) { var values = new List(); - var month = i < 10 ? $"0{i}" : i.ToString(); - var row = (startRow + i).ToString(); + var monthStr = month < 10 ? $"0{month}" : month.ToString(); + foreach (string code in codesRow) { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}{month}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}"); + values.Add(record?.Value1?.ToString() ?? "0"); } + var valueRange = new ValueRange { Values = new List> { values } }; - var update = _googleSheetValues.Update(valueRange, sheetId, $"{reportSheetName}!C{row}:XZ{row}"); + + var row = (startRow + month).ToString(); + var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); - } - // sum + _logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet", + ProcessorType, month); + } + } + + private void UpdateYearSummary(string sheetId, Layer processedLayer, IList codesRow) + { + const int startRow = 6; var valuesSum = new List(); var emptyRow = new List(); - var rowEmpty = (startRow + 13).ToString(); - var rowSum = (startRow + 14).ToString(); + foreach (string code in codesRow) { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } + valuesSum.Add(record?.Value1?.ToString() ?? "0"); } - // insert empty row before sum + + // Insert empty row before sum + var rowEmpty = (startRow + 13).ToString(); var valueRangeEmpty = new ValueRange { Values = new List> { emptyRow } }; - var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{reportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); + var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateEmpty.Execute(); + // Update sum row + var rowSum = (startRow + 14).ToString(); var valueRangeSum = new ValueRange { Values = new List> { valuesSum } }; - var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{reportSheetName}!C{rowSum}:XZ{rowSum}"); + var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}"); updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateSum.Execute(); - // update time - var timeUtc = new List - { - r2!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; + _logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + // Update UTC time + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeUtcTime = new ValueRange { - Values = new List> { timeUtc } + Values = new List> { new List { timeUtc } } }; - var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{reportSheetName}!G1"); + var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1"); updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeUtc.Execute(); + // Update Warsaw time var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r2.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeWarsawTime = new ValueRange { - Values = new List> { timeWarsaw } + Values = new List> { new List { timeWarsaw } } }; - var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{reportSheetName}!G2"); + var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2"); updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeWarsaw.Execute(); - //invoices + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } - var invoices = r2.Records!.Where(x => x.Code!.Length == 12) - .OrderByDescending(x => x.Code); + private void UpdateInvoicesData(string sheetId, Layer processedLayer) + { + var invoices = processedLayer.Records! + .Where(x => x.Code!.Length == 12) + .OrderByDescending(x => x.Code) + .ToList(); var invoicesValues = new List>(); var cleanUpValues = new List>(); + foreach (var invoice in invoices) { - var invoiceDate = - DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) - .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); + var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) + .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); + var invoiceRow = new List { - invoiceDate, - "", - invoice.Desc1!, - invoice.Value1! + invoiceDate, "", invoice.Desc1!, invoice.Value1! }; invoicesValues.Add(invoiceRow); - var cleanupRow = new List - { - "", "", "", "" - }; + var cleanupRow = new List { "", "", "", "" }; cleanUpValues.Add(cleanupRow); } - + // Clear existing data var cleanupValueRange = new ValueRange { Values = cleanUpValues }; - var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - cleanupInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; cleanupInvoices.Execute(); - + // Update with new data var invoicesValueRange = new ValueRange { Values = invoicesValues }; - var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - updateInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateInvoices.Execute(); + _logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet", + ProcessorType, invoices.Count); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs index f58f1ca..73368aa 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs @@ -1,8 +1,8 @@ using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; using Google.Apis.Sheets.v4; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Processors; @@ -14,6 +14,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + public T4SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,70 +32,196 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) + try { - throw new Exception("SourceLayer record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data source + var dataSource = GetDataSource(sourceImportWorker); + + // Process records (simple copy) + var newRecords = ProcessRecords(dataSource); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + if (sourceImportWorker == null) { - throw new Exception("SourceImportWorker layer not found"); + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); } - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T4"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } + private Layer GetDataSource(Layer sourceImportWorker) + { var dataSource = _db.Layers + .Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled) .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (dataSource == null) { - throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); + throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}"); } - var newRecords = dataSource.Records!.Select(record => new Record + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + + return dataSource; + } + + private List ProcessRecords(Layer dataSource) + { + if (dataSource.Records == null || dataSource.Records.Count == 0) + { + _logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType); + return new List(); + } + + var newRecords = dataSource.Records.Select(record => new Record { Id = Guid.NewGuid(), Code = record.Code, @@ -99,30 +231,46 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor ModifiedAt = DateTime.UtcNow }).ToList(); - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -133,6 +281,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 55dcf9b..8bc2d3f 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -13,6 +13,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + public T5LastValuesProcessor( AppDbContext db, ILogger logger) @@ -20,97 +26,279 @@ public class T5LastValuesProcessor : MorskaBaseProcessor _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) throw new Exception("SourceLayer record not found"); - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) throw new Exception("SourceImportWorker layer not found"); - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) throw new Exception("Source record not found"); + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(sourceImportWorker); + + // Process records (get last values for each code) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + + if (sourceImportWorker == null) + { + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); + } + + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T5"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources(Layer sourceImportWorker) + { var dataSources = _db.Layers .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == sourceImportWorker.Id && + !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); - if (dataSources.Count == 0) throw new Exception($"DataSource is empty, {sourceImportWorker.Name}"); + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}"); + } - var codes = dataSources.SelectMany(x => x.Records!).Select(x => x.Code).Distinct().ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}", + ProcessorType, dataSources.Count, sourceImportWorker.Name); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var codes = allRecords.Select(x => x.Code).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records", + ProcessorType, codes.Count, allRecords.Count); foreach (var code in codes) { - var lastRecord = dataSources.SelectMany(x => x.Records!).Where(x => x.Code == code).OrderByDescending(x => x.CreatedAt).FirstOrDefault(); + var lastRecord = allRecords + .Where(x => x.Code == code) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + if (lastRecord == null) continue; - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - for (var i = 1; i < 33; i++) - { - if (ProcessHelper.GetValue(lastRecord, i) != null) - { - ProcessHelper.SetValue(processedRecord, i, ProcessHelper.GetValue(lastRecord, i)); - } - } - + var processedRecord = CreateProcessedRecord(lastRecord); newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}", + ProcessorType, code, lastRecord.CreatedAt); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateProcessedRecord(Record lastRecord) + { + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = lastRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(lastRecord, i); + if (value != null) + { + ProcessHelper.SetValue(processedRecord, i, value); + } + } + + return processedRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } else + { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -121,6 +309,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file From 117a67ab582d7f84651f13439ed60d35776e8107 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 7 Jun 2025 16:34:36 +0200 Subject: [PATCH 08/11] Adjust processor type names --- ...or.cs => MorskaT3SingleSourceProcessor.cs} | 10 +-- .../Processors/T1R1Processor.cs | 2 +- .../Processors/T1R3Processor.cs | 2 +- ...ceCopySelectedCodesProcessor-TO_REMOVE.cs} | 0 ...tedCodesYearSummaryProcessor-TO_REMOVE.cs} | 0 .../T3MultiSourceSummaryProcessor.cs | 2 +- .../T3MultiSourceYearSummaryProcessor.cs | 2 +- .../T3SourceYearSummaryProcessor.cs | 2 +- .../Processors/T4R2Processor.cs | 2 +- .../Processors/T4SingleSourceProcessor.cs | 2 +- .../Processors/T5LastValuesProcessor.cs | 2 +- .../Controllers/LayersController.cs | 78 +++++++++++++------ tools/http-tests/AutoProcess.http | 2 +- tools/http-tests/BackupDatabase.http | 2 +- 14 files changed, 71 insertions(+), 37 deletions(-) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3SingleSourceProcessor.cs => MorskaT3SingleSourceProcessor.cs} (97%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceCopySelectedCodesProcessor.cs => T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs} (100%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs => T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs} (100%) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs similarity index 97% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs index 57c0fa6..15e7602 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SingleSourceProcessor.cs @@ -7,13 +7,13 @@ using Google.Apis.Sheets.v4; namespace DiunaBI.Plugins.Morska.Processors; -public class T3SingleSourceProcessor : MorskaBaseProcessor +public class MorskaT3SingleSourceProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T3.SingleSource"; + public override string ProcessorType => "Morska.Process.T3.SingleSource"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -22,10 +22,10 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor private string? Source { get; set; } private Layer? SourceImportWorker { get; set; } - public T3SingleSourceProcessor( + public MorskaT3SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index d5801e6..f7a2ebd 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -12,7 +12,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T1R1Processor : MorskaBaseProcessor { - public override string ProcessorType => "T1.R1"; + public override string ProcessorType => "Morska.Process.T1.R1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs index 7a4cc34..c61df8a 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs @@ -12,7 +12,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T1R3Processor : MorskaBaseProcessor { - public override string ProcessorType => "T1.R3"; + public override string ProcessorType => "Morska.Process.T1.R3"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs similarity index 100% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs similarity index 100% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index e7f642d..ce7247a 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T3.MultiSourceSummary"; + public override string ProcessorType => "Morska.Process.T3.MultiSourceSummary"; private readonly AppDbContext _db; private readonly ILogger _logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index 75b8f43..66d1a13 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T3.MultiSourceYearSummary"; + public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary"; private readonly AppDbContext _db; private readonly ILogger _logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 0fcd769..2b1f4da 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -9,7 +9,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3SourceYearSummaryProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T3.SourceYearSummary"; + public override string ProcessorType => "Morska.Process.T3.SourceYearSummary"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index e2fccf9..357e877 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -11,7 +11,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T4R2Processor : MorskaBaseProcessor { - public override string ProcessorType => "T4.R2"; + public override string ProcessorType => "Morska.Process.T4.R2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs index 73368aa..30cdab3 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T4SingleSourceProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T4.SingleSource"; + public override string ProcessorType => "Morska.Process.T4.SingleSource"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 8bc2d3f..4f290dd 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -8,7 +8,7 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T5LastValuesProcessor : MorskaBaseProcessor { - public override string ProcessorType => "T5.LastValues"; + public override string ProcessorType => "Morska.Process.T5.LastValues"; private readonly AppDbContext _db; private readonly ILogger _logger; diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 5288bd7..0158348 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -757,31 +757,66 @@ public class LayersController : Controller } } + [HttpGet] + [Route("CheckProcessors")] + [AllowAnonymous] + public IActionResult CheckProcessors() + { + // get list od all enabled processors and check if they has record 'Plugin' + var enabledProcessors = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") + ) + .OrderByDescending(x => x.CreatedAt) + .AsNoTracking() + .ToList(); + + _logger.LogInformation("CheckProcessors: Found {ProcessorCount} enabled processors", enabledProcessors.Count); + foreach (var processor in enabledProcessors) + { + var pluginRecord = processor.Records!.FirstOrDefault(x => x.Code == "Plugin"); + if (pluginRecord == null) + { + _logger.LogWarning("CheckProcessors: No Plugin record found for processor {ProcessorName} ({ProcessorId}), skipping", + processor.Name, processor.Id); + continue; + } + + var pluginName = pluginRecord.Desc1; + if (string.IsNullOrEmpty(pluginName)) + { + _logger.LogWarning("CheckProcessors: Empty Plugin name for processor {ProcessorName} ({ProcessorId}), skipping", + processor.Name, processor.Id); + continue; + } + + var processorInstance = _pluginManager.GetProcessor(pluginName); + if (processorInstance == null) + { + _logger.LogWarning("CheckProcessors: Processor {PluginName} not found for {ProcessorName} ({ProcessorId}), skipping", + pluginName, processor.Name, processor.Id); + continue; + } + } + _logger.LogInformation("CheckProcessors: Completed checking processors"); + + return Ok(); + } + [HttpGet] [Route("AddPluginName")] [AllowAnonymous] public IActionResult AddPluginName() { - var record = new Record - { - Id = Guid.NewGuid(), - LayerId = Guid.Parse("eb5b4d0e-1607-4445-bbe5-65b9b8416787"), - Code = "Plugin", - Desc1 = "Morska.Import.Standard", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") - }; - _db.Records.Add(record); - _db.SaveChanges(); - - /* + var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && + x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == "T1-R3") ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() @@ -794,21 +829,20 @@ public class LayersController : Controller Id = Guid.NewGuid(), LayerId = importWorker.Id, Code = "Plugin", - Desc1 = "Morska.Import.D3", + Desc1 = "Morska.Process.T1.R3", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") }; - //_db.Records.Add(record); + _db.Records.Add(record); } - //_db.SaveChanges(); - */ + _db.SaveChanges(); + return Ok(); } - private static void WriteToConsole(params string[] messages) { foreach (var message in messages) diff --git a/tools/http-tests/AutoProcess.http b/tools/http-tests/AutoProcess.http index 97419fc..c807c75 100644 --- a/tools/http-tests/AutoProcess.http +++ b/tools/http-tests/AutoProcess.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AutoProcess/10763478CB738D4ecb2h76g803478CB738D4e +GET http://localhost:5400/api/Layers/CheckProcessors Timeout: 500000 diff --git a/tools/http-tests/BackupDatabase.http b/tools/http-tests/BackupDatabase.http index 539564b..043e64e 100644 --- a/tools/http-tests/BackupDatabase.http +++ b/tools/http-tests/BackupDatabase.http @@ -1,2 +1,2 @@ ### -GET http://localhost:5400/api/Admin/BackupDatabase/10763478CB738D4ecb2h76g803478CB738D4e +GET http://localhost:5400/api/Layers/AddPluginName From ff41a71484d8bd11ee43383cf6f2af781dc1a689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sun, 8 Jun 2025 10:18:52 +0200 Subject: [PATCH 09/11] Final fixes before tests --- .../Services/JobQueueProcessor.cs | 110 +++-- .../Processors/T1R1Processor.cs | 4 +- .../T3MultiSourceSummaryProcessor.cs | 4 +- .../T3MultiSourceYearSummaryProcessor.cs | 4 +- .../T3SourceYearSummaryProcessor.cs | 4 +- .../Processors/T4R2Processor.cs | 4 +- .../Processors/T5LastValuesProcessor.cs | 2 +- .../Controllers/LayersController.cs | 418 ++++++++++++------ src/Backend/DiunaBI.WebAPI/Program.cs | 23 +- tools/http-tests/AutoImport.http | 2 +- 10 files changed, 374 insertions(+), 201 deletions(-) diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs index 66c7387..624441c 100644 --- a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs +++ b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs @@ -17,6 +17,7 @@ public class JobQueueProcessor : BackgroundService { private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; + private readonly ManualResetEventSlim _processSignal; public JobQueueProcessor( IServiceProvider serviceProvider, @@ -24,27 +25,30 @@ public class JobQueueProcessor : BackgroundService { _serviceProvider = serviceProvider; _logger = logger; + _processSignal = new ManualResetEventSlim(false); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - _logger.LogInformation("JobQueueProcessor: Started"); + _logger.LogInformation("JobQueueProcessor: Started (manual trigger mode)"); while (!stoppingToken.IsCancellationRequested) { try { + // Wait for manual trigger or cancellation + _processSignal.Wait(stoppingToken); + _processSignal.Reset(); + + _logger.LogInformation("JobQueueProcessor: Processing triggered manually"); + using var scope = _serviceProvider.CreateScope(); var queueService = scope.ServiceProvider.GetRequiredService(); - // First process all imports (they run sequentially due to Google Sheets API limits) - await ProcessJobType(queueService, JobType.Import, maxConcurrency: 1, stoppingToken); + // Process all jobs until queue is empty + await ProcessAllJobs(queueService, stoppingToken); - // Then process processors (can run in parallel within same priority) - await ProcessJobType(queueService, JobType.Process, maxConcurrency: 3, stoppingToken); - - // Wait before next cycle - await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + _logger.LogInformation("JobQueueProcessor: Manual processing completed"); } catch (OperationCanceledException) { @@ -54,57 +58,79 @@ public class JobQueueProcessor : BackgroundService catch (Exception ex) { _logger.LogError(ex, "JobQueueProcessor: Unexpected error in queue processor"); - await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken); + await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken); } } _logger.LogInformation("JobQueueProcessor: Stopped"); } - private async Task ProcessJobType(IJobQueueService queueService, JobType jobType, int maxConcurrency, CancellationToken cancellationToken) + public void TriggerProcessing() { - var runningJobs = await queueService.GetRunningJobsCountAsync(jobType); + _logger.LogInformation("JobQueueProcessor: Manual trigger received"); + _processSignal.Set(); + } - // Don't start new jobs if we're at max concurrency - if (runningJobs >= maxConcurrency) + private async Task ProcessAllJobs(IJobQueueService queueService, CancellationToken cancellationToken) + { + var startTime = DateTime.UtcNow; + var initialQueueSize = await queueService.GetQueueCountAsync(); + int processedJobs = 0; + int failedJobs = 0; + + _logger.LogInformation("JobQueueProcessor: Starting processing of {InitialQueueSize} jobs", initialQueueSize); + + while (!cancellationToken.IsCancellationRequested) { - return; - } - - var job = await queueService.DequeueJobAsync(jobType); - if (job == null) - { - return; - } - - _logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})", - job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority); - - // Process job asynchronously to allow parallel processing of processors - _ = Task.Run(async () => - { - try + // First process all imports (they run sequentially due to Google Sheets API limits) + var importJob = await queueService.DequeueJobAsync(JobType.Import); + if (importJob != null) { - await ProcessJobAsync(job, cancellationToken); + await ProcessJobAsync(importJob, cancellationToken); + if (importJob.Status == JobStatus.Completed) processedJobs++; + else failedJobs++; // Add delay between imports to respect Google Sheets API limits - if (job.JobType == JobType.Import) - { - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - } + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + continue; } - catch (Exception ex) + + // Then process processors (can run in parallel within same priority) + var processJob = await queueService.DequeueJobAsync(JobType.Process); + if (processJob != null) { - _logger.LogError(ex, "JobQueueProcessor: Error in background job processing for {JobType} job {JobId}", - job.JobType, job.Id); + await ProcessJobAsync(processJob, cancellationToken); + if (processJob.Status == JobStatus.Completed) processedJobs++; + else failedJobs++; + continue; } - }, cancellationToken); + + // No more jobs in queue + break; + } + + var endTime = DateTime.UtcNow; + var duration = endTime - startTime; + var finalQueueSize = await queueService.GetQueueCountAsync(); + + _logger.LogInformation("JobQueueProcessor: Processing completed. Duration: {Duration:hh\\:mm\\:ss}, " + + "Initial queue: {InitialQueueSize}, Processed: {ProcessedJobs}, Failed: {FailedJobs}, " + + "Final queue size: {FinalQueueSize}", + duration, initialQueueSize, processedJobs, failedJobs, finalQueueSize); + + if (failedJobs > 0) + { + _logger.LogWarning("JobQueueProcessor: {FailedJobs} jobs failed during processing. Check logs for details.", failedJobs); + } } private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken) { try { + _logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})", + job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority); + using var scope = _serviceProvider.CreateScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var pluginManager = scope.ServiceProvider.GetRequiredService(); @@ -119,6 +145,7 @@ public class JobQueueProcessor : BackgroundService { _logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId); await queueService.MarkJobFailedAsync(job.Id, "Layer not found"); + job.Status = JobStatus.Failed; return; } @@ -131,6 +158,7 @@ public class JobQueueProcessor : BackgroundService { _logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName); await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found"); + job.Status = JobStatus.Failed; return; } @@ -146,10 +174,11 @@ public class JobQueueProcessor : BackgroundService { _logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName); await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found"); + job.Status = JobStatus.Failed; return; } - _logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with plugin {PluginName}", + _logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with processor {PluginName}", layer.Name, job.PluginName); processor.Process(layer); @@ -160,6 +189,7 @@ public class JobQueueProcessor : BackgroundService } await queueService.MarkJobCompletedAsync(job.Id); + job.Status = JobStatus.Completed; _logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}", job.JobType, layer.Name); @@ -176,10 +206,12 @@ public class JobQueueProcessor : BackgroundService if (IsRetriableError(ex)) { await queueService.MarkJobForRetryAsync(job.Id, ex.Message); + job.Status = JobStatus.Retrying; } else { await queueService.MarkJobFailedAsync(job.Id, ex.Message); + job.Status = JobStatus.Failed; } } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index f7a2ebd..7b818b3 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -237,8 +237,8 @@ public class T1R1Processor : MorskaBaseProcessor { var dataSource = _db.Layers .Where(x => x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index ce7247a..ff5198c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -181,8 +181,8 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor var dataSources = Sources!.Select(source => _db.Layers .Where(x => x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index 66d1a13..c50cffe 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -171,8 +171,8 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor var dataSources = Sources!.Select(source => _db.Layers .Where(x => x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3")) + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 2b1f4da..00c0343 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -163,8 +163,8 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor { var dataSource = _db.Layers .Where(x => x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3")) + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index 357e877..6f5271d 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -287,8 +287,8 @@ public class T4R2Processor : MorskaBaseProcessor { return _db.Layers .Where(x => x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 4f290dd..7354601 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -195,7 +195,7 @@ public class T5LastValuesProcessor : MorskaBaseProcessor var dataSources = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == sourceImportWorker.Id && - !x.IsDeleted && !x.IsCancelled) + !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 0158348..bc60ece 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -676,147 +676,19 @@ public class LayersController : Controller } } } - - [HttpGet] - [Route("GetImportWorkers")] - [AllowAnonymous] - public async Task GetImportWorkers() - { - try - { - var importWorkerLayers = await _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - && x.Number == 5579 - ) - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToListAsync(); - - _logger.LogInformation("GetImportWorkers: Found {LayerCount} import worker layers to queue", - importWorkerLayers.Count); - - int queuedCount = 0; - - foreach (var importWorker in importWorkerLayers) - { - var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; - if (string.IsNullOrEmpty(pluginName)) - { - _logger.LogWarning("GetImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", - importWorker.Name, importWorker.Id); - continue; - } - - // Check if plugin exists - var importer = _pluginManager.GetImporter(pluginName); - if (importer == null) - { - _logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping", - pluginName, importWorker.Name, importWorker.Id); - continue; - } - - var job = new QueueJob - { - LayerId = importWorker.Id, - LayerName = importWorker.Name ?? "Unknown", - PluginName = pluginName, - JobType = JobType.Import, - Priority = 0, // All imports have same priority - MaxRetries = 5, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") - }; - - await _queueService.EnqueueJobAsync(job); - queuedCount++; - - _logger.LogDebug("GetImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}", - importWorker.Name, importWorker.Id, pluginName); - } - - var totalQueueSize = await _queueService.GetQueueCountAsync(); - - _logger.LogInformation("GetImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}", - queuedCount, totalQueueSize); - - return Ok(new { - Message = $"Queued {queuedCount} import jobs", - QueuedJobs = queuedCount, - TotalQueueSize = totalQueueSize, - SkippedLayers = importWorkerLayers.Count - queuedCount - }); - } - catch (Exception e) - { - _logger.LogError(e, "GetImportWorkers: Error queuing import workers"); - return BadRequest(e.ToString()); - } - } - [HttpGet] [Route("CheckProcessors")] [AllowAnonymous] public IActionResult CheckProcessors() { - // get list od all enabled processors and check if they has record 'Plugin' - var enabledProcessors = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - ) - .OrderByDescending(x => x.CreatedAt) - .AsNoTracking() - .ToList(); + // get - _logger.LogInformation("CheckProcessors: Found {ProcessorCount} enabled processors", enabledProcessors.Count); - foreach (var processor in enabledProcessors) - { - var pluginRecord = processor.Records!.FirstOrDefault(x => x.Code == "Plugin"); - if (pluginRecord == null) - { - _logger.LogWarning("CheckProcessors: No Plugin record found for processor {ProcessorName} ({ProcessorId}), skipping", - processor.Name, processor.Id); - continue; - } - - var pluginName = pluginRecord.Desc1; - if (string.IsNullOrEmpty(pluginName)) - { - _logger.LogWarning("CheckProcessors: Empty Plugin name for processor {ProcessorName} ({ProcessorId}), skipping", - processor.Name, processor.Id); - continue; - } - - var processorInstance = _pluginManager.GetProcessor(pluginName); - if (processorInstance == null) - { - _logger.LogWarning("CheckProcessors: Processor {PluginName} not found for {ProcessorName} ({ProcessorId}), skipping", - pluginName, processor.Name, processor.Id); - continue; - } - } - _logger.LogInformation("CheckProcessors: Completed checking processors"); - - return Ok(); - } - - [HttpGet] - [Route("AddPluginName")] - [AllowAnonymous] - public IActionResult AddPluginName() - { - + /* var importWorkerLayers = _db.Layers .Include(x => x.Records) .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && - x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == "T1-R3") + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") ) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() @@ -829,20 +701,21 @@ public class LayersController : Controller Id = Guid.NewGuid(), LayerId = importWorker.Id, Code = "Plugin", - Desc1 = "Morska.Process.T1.R3", + Desc1 = "Morska.Import.D3", CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow, CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") }; - _db.Records.Add(record); + //_db.Records.Add(record); } - _db.SaveChanges(); - + //_db.SaveChanges(); + */ return Ok(); } + private static void WriteToConsole(params string[] messages) { foreach (var message in messages) @@ -936,4 +809,277 @@ public class LayersController : Controller throw; } } + + [HttpGet] + [Route("EnqueueImportWorkers/{apiKey}")] + [AllowAnonymous] + public async Task EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null) + { + if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) + { + _logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); + return Unauthorized(); + } + + try + { + var query = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && + !x.IsDeleted && !x.IsCancelled + ); + + // If specific layerId is provided, filter to that layer only + if (layerId.HasValue) + { + query = query.Where(x => x.Id == layerId.Value); + } + + var importWorkerLayers = await query + .OrderBy(x => x.CreatedAt) + .AsNoTracking() + .ToListAsync(); + + _logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}", + importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); + + if (importWorkerLayers.Count == 0) + { + return Ok(new + { + Message = "No import workers found to queue", + QueuedJobs = 0, + TotalQueueSize = await _queueService.GetQueueCountAsync(), + SkippedLayers = 0 + }); + } + + int queuedCount = 0; + + foreach (var importWorker in importWorkerLayers) + { + var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; + if (string.IsNullOrEmpty(pluginName)) + { + _logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", + importWorker.Name, importWorker.Id); + continue; + } + + // Check if plugin exists + var importer = _pluginManager.GetImporter(pluginName); + if (importer == null) + { + _logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping", + pluginName, importWorker.Name, importWorker.Id); + continue; + } + + var job = new QueueJob + { + LayerId = importWorker.Id, + LayerName = importWorker.Name ?? "Unknown", + PluginName = pluginName, + JobType = JobType.Import, + Priority = 0, // All imports have same priority + MaxRetries = 5, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + + await _queueService.EnqueueJobAsync(job); + queuedCount++; + + _logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}", + importWorker.Name, importWorker.Id, pluginName); + } + + var totalQueueSize = await _queueService.GetQueueCountAsync(); + + _logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}", + queuedCount, totalQueueSize); + + return Ok(new + { + Message = $"Queued {queuedCount} import jobs", + QueuedJobs = queuedCount, + TotalQueueSize = totalQueueSize, + SkippedLayers = importWorkerLayers.Count - queuedCount + }); + } + catch (Exception e) + { + _logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers"); + return BadRequest(e.ToString()); + } + } + + [HttpGet] + [Route("EnqueueProcessWorkers/{apiKey}")] + [AllowAnonymous] + public async Task EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null) + { + if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) + { + _logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); + return Unauthorized(); + } + + try + { + var query = _db.Layers + .Include(x => x.Records) + .Where(x => + x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && + x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && + !x.IsDeleted && !x.IsCancelled + ); + + // If specific layerId is provided, filter to that layer only + if (layerId.HasValue) + { + query = query.Where(x => x.Id == layerId.Value); + } + + var processWorkerLayers = await query + .OrderBy(x => x.CreatedAt) + .AsNoTracking() + .ToListAsync(); + + _logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}", + processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); + + if (processWorkerLayers.Count == 0) + { + return Ok(new + { + Message = "No process workers found to queue", + QueuedJobs = 0, + TotalQueueSize = await _queueService.GetQueueCountAsync(), + SkippedLayers = 0 + }); + } + + int queuedCount = 0; + + foreach (var processWorker in processWorkerLayers) + { + var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; + if (string.IsNullOrEmpty(pluginName)) + { + _logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", + processWorker.Name, processWorker.Id); + continue; + } + + var processorType = processWorker.Records!.FirstOrDefault(x => x.Code == "ProcessorType")?.Desc1; + if (string.IsNullOrEmpty(processorType)) + { + _logger.LogWarning("EnqueueProcessWorkers: No processor type found for layer {LayerName} ({LayerId}), skipping", + processWorker.Name, processWorker.Id); + continue; + } + + // Check if processor exists + var processor = _pluginManager.GetProcessor(processorType); + if (processor == null) + { + _logger.LogWarning("EnqueueProcessWorkers: Processor {ProcessorType} not found for layer {LayerName} ({LayerId}), skipping", + processorType, processWorker.Name, processWorker.Id); + continue; + } + + // Get priority from ProcessWorker record, default to 10 if not found + var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1; + var priority = 10; // Default priority + if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority)) + { + priority = parsedPriority; + } + + var job = new QueueJob + { + LayerId = processWorker.Id, + LayerName = processWorker.Name ?? "Unknown", + PluginName = processorType, // Use processorType as PluginName for process jobs + JobType = JobType.Process, + Priority = priority, + MaxRetries = 3, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") + }; + + await _queueService.EnqueueJobAsync(job); + queuedCount++; + + _logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with processor {ProcessorType}, priority {Priority}", + processWorker.Name, processWorker.Id, processorType, priority); + } + + var totalQueueSize = await _queueService.GetQueueCountAsync(); + + _logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}", + queuedCount, totalQueueSize); + + return Ok(new + { + Message = $"Queued {queuedCount} process jobs", + QueuedJobs = queuedCount, + TotalQueueSize = totalQueueSize, + SkippedLayers = processWorkerLayers.Count - queuedCount + }); + } + catch (Exception e) + { + _logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers"); + return BadRequest(e.ToString()); + } + } + + [HttpGet] + [Route("RunQueueJobs/{apiKey}")] + [AllowAnonymous] + public async Task RunQueueJobs(string apiKey) + { + if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) + { + _logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey); + return Unauthorized(); + } + + try + { + var queueSize = await _queueService.GetQueueCountAsync(); + + if (queueSize == 0) + { + return Ok(new + { + Message = "Queue is empty", + QueueSize = 0, + Status = "No jobs to process" + }); + } + + _logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize); + + // PRZYWRÓĆ SINGLETON ACCESS: + var queueProcessor = HttpContext.RequestServices.GetRequiredService(); + queueProcessor.TriggerProcessing(); + + return Ok(new + { + Message = $"Queue processing triggered for {queueSize} jobs", + QueueSize = queueSize, + Status = "Processing started in background" + }); + } + catch (Exception e) + { + _logger.LogError(e, "RunQueueJobs: Error triggering queue processing"); + return BadRequest(e.ToString()); + } + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 2f29ace..99b7654 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -1,5 +1,3 @@ -using Google.Apis.Auth.OAuth2; -using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; @@ -14,7 +12,6 @@ using DiunaBI.Core.Interfaces; var builder = WebApplication.CreateBuilder(args); -// ✅ SERILOG TYLKO DLA PRODUKCJI if (builder.Environment.IsProduction()) { builder.Host.UseSerilog((context, configuration) => @@ -30,6 +27,7 @@ if (builder.Environment.IsProduction()) } var connectionString = builder.Configuration.GetConnectionString("SQLDatabase"); + builder.Services.AddDbContext(x => { x.UseSqlServer(connectionString); @@ -70,25 +68,24 @@ builder.Services.AddAuthentication(options => IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!)) }; }); -builder.Services.AddAuthentication(); // Queue services builder.Services.AddScoped(); -builder.Services.AddHostedService(); +builder.Services.AddSingleton(); -// Zarejestruj Google Sheets dependencies +// Google Sheets dependencies builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(provider => { var googleSheetsHelper = provider.GetRequiredService(); var valuesResource = googleSheetsHelper.Service?.Spreadsheets.Values; - + if (valuesResource == null) { throw new InvalidOperationException("Google Sheets Service is not initialized properly"); } - + return valuesResource; }); @@ -96,7 +93,6 @@ builder.Services.AddSingleton(); var app = builder.Build(); -// ✅ SERILOG REQUEST LOGGING TYLKO DLA PRODUKCJI if (app.Environment.IsProduction()) { app.UseSerilogRequestLogging(options => @@ -106,25 +102,25 @@ if (app.Environment.IsProduction()) { diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); - + var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); if (!string.IsNullOrEmpty(userAgent)) { diagnosticContext.Set("UserAgent", userAgent); } - + diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); }; }); } +// Plugin initialization var pluginManager = app.Services.GetRequiredService(); var executablePath = Assembly.GetExecutingAssembly().Location; var executableDir = Path.GetDirectoryName(executablePath)!; var pluginsPath = Path.Combine(executableDir, "Plugins"); -// ✅ RÓŻNE LOGGERY W ZALEŻNOŚCI OD ŚRODOWISKA if (app.Environment.IsProduction()) { Log.Information("Starting DiunaBI application"); @@ -165,8 +161,7 @@ app.MapControllers(); app.Run(); -// ✅ SERILOG CLEANUP TYLKO DLA PRODUKCJI if (app.Environment.IsProduction()) { Log.CloseAndFlush(); -} +} \ No newline at end of file diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index 53b13da..a5f2d95 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/GetImportWorkers +GET http://localhost:5400/api/Layers/AddPriorityRecords From 99d8593c49a8a78ed95cae8e3c357c5da38ae53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sun, 8 Jun 2025 10:56:20 +0200 Subject: [PATCH 10/11] Naming fix --- .../Services/JobQueueProcessor.cs | 38 ++++++++++-------- ...1R1Processor.cs => MorskaT1R1Processor.cs} | 8 ++-- ...1R3Processor.cs => MorskaT1R3Processor.cs} | 8 ++-- ...ceCopySelectedCodesProcessor-TO_REMOVE.cs} | 8 ++-- ...tedCodesYearSummaryProcessor-TO_REMOVE.cs} | 8 ++-- ...=> MorskaT3MultiSourceSummaryProcessor.cs} | 8 ++-- ...orskaT3MultiSourceYearSummaryProcessor.cs} | 8 ++-- ... => MorskaT3SourceYearSummaryProcessor.cs} | 8 ++-- ...4R2Processor.cs => MorskaT4R2Processor.cs} | 8 ++-- ...or.cs => MorskaT4SingleSourceProcessor.cs} | 8 ++-- ...ssor.cs => MorskaT5LastValuesProcessor.cs} | 8 ++-- .../Controllers/LayersController.cs | 39 ++++++++++--------- src/Backend/DiunaBI.WebAPI/Program.cs | 3 +- tools/http-tests/AutoImport.http | 2 +- 14 files changed, 84 insertions(+), 78 deletions(-) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T1R1Processor.cs => MorskaT1R1Processor.cs} (98%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T1R3Processor.cs => MorskaT1R3Processor.cs} (98%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs => MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs} (97%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs => MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs} (96%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceSummaryProcessor.cs => MorskaT3MultiSourceSummaryProcessor.cs} (98%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3MultiSourceYearSummaryProcessor.cs => MorskaT3MultiSourceYearSummaryProcessor.cs} (97%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T3SourceYearSummaryProcessor.cs => MorskaT3SourceYearSummaryProcessor.cs} (97%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T4R2Processor.cs => MorskaT4R2Processor.cs} (99%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T4SingleSourceProcessor.cs => MorskaT4SingleSourceProcessor.cs} (97%) rename src/Backend/DiunaBI.Plugins.Morska/Processors/{T5LastValuesProcessor.cs => MorskaT5LastValuesProcessor.cs} (98%) diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs index 624441c..2ea61c5 100644 --- a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs +++ b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs @@ -15,17 +15,20 @@ namespace DiunaBI.Core.Services; public class JobQueueProcessor : BackgroundService { - private readonly IServiceProvider _serviceProvider; + private readonly IServiceScopeFactory _scopeFactory; // ✅ GOOD - używa scope factory private readonly ILogger _logger; - private readonly ManualResetEventSlim _processSignal; + private readonly ManualResetEventSlim _processSignal = new(false); + + // ❌ USUŃ DIRECT INJECTION scoped services: + // private readonly IJobQueueService _queueService; + // private readonly AppDbContext _db; public JobQueueProcessor( - IServiceProvider serviceProvider, + IServiceScopeFactory scopeFactory, // ✅ GOOD - inject scope factory ILogger logger) { - _serviceProvider = serviceProvider; + _scopeFactory = scopeFactory; _logger = logger; - _processSignal = new ManualResetEventSlim(false); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -36,19 +39,18 @@ public class JobQueueProcessor : BackgroundService { try { - // Wait for manual trigger or cancellation + // Wait for manual trigger _processSignal.Wait(stoppingToken); _processSignal.Reset(); - _logger.LogInformation("JobQueueProcessor: Processing triggered manually"); + _logger.LogInformation("JobQueueProcessor: Processing triggered"); - using var scope = _serviceProvider.CreateScope(); + // ✅ GOOD - create scope for each processing cycle + using var scope = _scopeFactory.CreateScope(); var queueService = scope.ServiceProvider.GetRequiredService(); + var pluginManager = scope.ServiceProvider.GetRequiredService(); - // Process all jobs until queue is empty - await ProcessAllJobs(queueService, stoppingToken); - - _logger.LogInformation("JobQueueProcessor: Manual processing completed"); + await ProcessQueueAsync(queueService, pluginManager, stoppingToken); } catch (OperationCanceledException) { @@ -57,8 +59,8 @@ public class JobQueueProcessor : BackgroundService } catch (Exception ex) { - _logger.LogError(ex, "JobQueueProcessor: Unexpected error in queue processor"); - await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken); + _logger.LogError(ex, "JobQueueProcessor: Error in processing loop"); + await Task.Delay(5000, stoppingToken); // Wait before retry } } @@ -71,7 +73,7 @@ public class JobQueueProcessor : BackgroundService _processSignal.Set(); } - private async Task ProcessAllJobs(IJobQueueService queueService, CancellationToken cancellationToken) + private async Task ProcessQueueAsync(IJobQueueService queueService, PluginManager pluginManager, CancellationToken cancellationToken) { var startTime = DateTime.UtcNow; var initialQueueSize = await queueService.GetQueueCountAsync(); @@ -131,7 +133,8 @@ public class JobQueueProcessor : BackgroundService _logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})", job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority); - using var scope = _serviceProvider.CreateScope(); + // POPRAWKA: używaj _scopeFactory zamiast _serviceProvider + using var scope = _scopeFactory.CreateScope(); var dbContext = scope.ServiceProvider.GetRequiredService(); var pluginManager = scope.ServiceProvider.GetRequiredService(); var queueService = scope.ServiceProvider.GetRequiredService(); @@ -199,7 +202,8 @@ public class JobQueueProcessor : BackgroundService _logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}", job.JobType, job.Id, job.LayerName); - using var scope = _serviceProvider.CreateScope(); + // POPRAWKA: tutaj też używaj _scopeFactory + using var scope = _scopeFactory.CreateScope(); var queueService = scope.ServiceProvider.GetRequiredService(); // Check if it's a retriable error diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs similarity index 98% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs index 7b818b3..16d02a0 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs @@ -10,13 +10,13 @@ using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; -public class T1R1Processor : MorskaBaseProcessor +public class MorskaT1R1Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T1.R1"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -24,10 +24,10 @@ public class T1R1Processor : MorskaBaseProcessor private List? DynamicCodes { get; set; } private string? GoogleSheetName { get; set; } - public T1R1Processor( + public MorskaT1R1Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs similarity index 98% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs index c61df8a..745d1f1 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs @@ -10,22 +10,22 @@ using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; -public class T1R3Processor : MorskaBaseProcessor +public class MorskaT1R3Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T1.R3"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private string? Source { get; set; } - public T1R3Processor( + public MorskaT1R3Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs similarity index 97% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs index 11fce8d..cccb5da 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesProcessor-TO_REMOVE.cs @@ -6,12 +6,12 @@ using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; -public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor +public class MorskaT3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.MultiSourceCopySelectedCodes"; private readonly AppDbContext _db; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -20,9 +20,9 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor private string? Codes { get; set; } private List? CodesList { get; set; } - public T3MultiSourceCopySelectedCodesProcessor( + public MorskaT3MultiSourceCopySelectedCodesProcessor( AppDbContext db, - ILogger logger) + ILogger logger) { _db = db; _logger = logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs similarity index 96% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs index dd7f50c..b309ab2 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor-TO_REMOVE.cs @@ -6,19 +6,19 @@ using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; -public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor +public class MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary"; private readonly AppDbContext _db; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } - public T3MultiSourceCopySelectedCodesYearSummaryProcessor( + public MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor( AppDbContext db, - ILogger logger) + ILogger logger) { _db = db; _logger = logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs similarity index 98% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs index ff5198c..e5340e4 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceSummaryProcessor.cs @@ -7,12 +7,12 @@ using DiunaBI.Core.Services.Calculations; namespace DiunaBI.Plugins.Morska.Processors; -public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor +public class MorskaT3MultiSourceSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T3.MultiSourceSummary"; private readonly AppDbContext _db; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -20,9 +20,9 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor private List? Sources { get; set; } private List? DynamicCodes { get; set; } - public T3MultiSourceSummaryProcessor( + public MorskaT3MultiSourceSummaryProcessor( AppDbContext db, - ILogger logger) + ILogger logger) { _db = db; _logger = logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs similarity index 97% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs index c50cffe..0b8642b 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3MultiSourceYearSummaryProcessor.cs @@ -7,21 +7,21 @@ using DiunaBI.Core.Services.Calculations; namespace DiunaBI.Plugins.Morska.Processors; -public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor +public class MorskaT3MultiSourceYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary"; private readonly AppDbContext _db; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private List? Sources { get; set; } private List? DynamicCodes { get; set; } - public T3MultiSourceYearSummaryProcessor( + public MorskaT3MultiSourceYearSummaryProcessor( AppDbContext db, - ILogger logger) + ILogger logger) { _db = db; _logger = logger; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs similarity index 97% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs index 00c0343..2280cc7 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT3SourceYearSummaryProcessor.cs @@ -7,22 +7,22 @@ using DiunaBI.Core.Services; namespace DiunaBI.Plugins.Morska.Processors; -public class T3SourceYearSummaryProcessor : MorskaBaseProcessor +public class MorskaT3SourceYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T3.SourceYearSummary"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } private string? Source { get; set; } - public T3SourceYearSummaryProcessor( + public MorskaT3SourceYearSummaryProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs similarity index 99% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs index 6f5271d..81d49b1 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs @@ -9,13 +9,13 @@ using DiunaBI.Core.Services; namespace DiunaBI.Plugins.Morska.Processors; -public class T4R2Processor : MorskaBaseProcessor +public class MorskaT4R2Processor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T4.R2"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -24,10 +24,10 @@ public class T4R2Processor : MorskaBaseProcessor private string? ReportSheetName { get; set; } private string? InvoicesSheetName { get; set; } - public T4R2Processor( + public MorskaT4R2Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs similarity index 97% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs index 30cdab3..c4bd66e 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT4SingleSourceProcessor.cs @@ -6,13 +6,13 @@ using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Processors; -public class T4SingleSourceProcessor : MorskaBaseProcessor +public class MorskaT4SingleSourceProcessor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T4.SingleSource"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -20,10 +20,10 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor private string? SourceLayer { get; set; } private string? Source { get; set; } - public T4SingleSourceProcessor( + public MorskaT4SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs similarity index 98% rename from src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs rename to src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs index 7354601..0d84c0c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs @@ -6,12 +6,12 @@ using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; -public class T5LastValuesProcessor : MorskaBaseProcessor +public class MorskaT5LastValuesProcessor : MorskaBaseProcessor { public override string ProcessorType => "Morska.Process.T5.LastValues"; private readonly AppDbContext _db; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties loaded from layer records private int Year { get; set; } @@ -19,9 +19,9 @@ public class T5LastValuesProcessor : MorskaBaseProcessor private string? SourceLayer { get; set; } private string? Source { get; set; } - public T5LastValuesProcessor( + public MorskaT5LastValuesProcessor( AppDbContext db, - ILogger logger) + ILogger logger) { _db = db; _logger = logger; diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index bc60ece..2fca29c 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -829,7 +829,7 @@ public class LayersController : Controller x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && !x.IsDeleted && !x.IsCancelled - ); + ).Take(5); // If specific layerId is provided, filter to that layer only if (layerId.HasValue) @@ -935,7 +935,7 @@ public class LayersController : Controller x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && !x.IsDeleted && !x.IsCancelled - ); + ).Take(5); // If specific layerId is provided, filter to that layer only if (layerId.HasValue) @@ -966,6 +966,7 @@ public class LayersController : Controller foreach (var processWorker in processWorkerLayers) { + // POPRAWIONE: Używaj Plugin zamiast ProcessorType var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; if (string.IsNullOrEmpty(pluginName)) { @@ -974,20 +975,12 @@ public class LayersController : Controller continue; } - var processorType = processWorker.Records!.FirstOrDefault(x => x.Code == "ProcessorType")?.Desc1; - if (string.IsNullOrEmpty(processorType)) - { - _logger.LogWarning("EnqueueProcessWorkers: No processor type found for layer {LayerName} ({LayerId}), skipping", - processWorker.Name, processWorker.Id); - continue; - } - - // Check if processor exists - var processor = _pluginManager.GetProcessor(processorType); + // POPRAWIONE: Sprawdź czy processor istnieje przez pluginName + var processor = _pluginManager.GetProcessor(pluginName); if (processor == null) { - _logger.LogWarning("EnqueueProcessWorkers: Processor {ProcessorType} not found for layer {LayerName} ({LayerId}), skipping", - processorType, processWorker.Name, processWorker.Id); + _logger.LogWarning("EnqueueProcessWorkers: Processor {PluginName} not found for layer {LayerName} ({LayerId}), skipping", + pluginName, processWorker.Name, processWorker.Id); continue; } @@ -1003,7 +996,7 @@ public class LayersController : Controller { LayerId = processWorker.Id, LayerName = processWorker.Name ?? "Unknown", - PluginName = processorType, // Use processorType as PluginName for process jobs + PluginName = pluginName, // POPRAWIONE: Używaj pluginName bezpośrednio JobType = JobType.Process, Priority = priority, MaxRetries = 3, @@ -1014,8 +1007,8 @@ public class LayersController : Controller await _queueService.EnqueueJobAsync(job); queuedCount++; - _logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with processor {ProcessorType}, priority {Priority}", - processWorker.Name, processWorker.Id, processorType, priority); + _logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with plugin {PluginName}, priority {Priority}", + processWorker.Name, processWorker.Id, pluginName, priority); } var totalQueueSize = await _queueService.GetQueueCountAsync(); @@ -1065,8 +1058,16 @@ public class LayersController : Controller _logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize); - // PRZYWRÓĆ SINGLETON ACCESS: - var queueProcessor = HttpContext.RequestServices.GetRequiredService(); + // ZMIEŃ NA DOSTĘP PRZEZ IHostedService: + var hostedServices = HttpContext.RequestServices.GetServices(); + var queueProcessor = hostedServices.OfType().FirstOrDefault(); + + if (queueProcessor == null) + { + _logger.LogError("RunQueueJobs: JobQueueProcessor not found"); + return BadRequest("JobQueueProcessor not found"); + } + queueProcessor.TriggerProcessing(); return Ok(new diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 99b7654..bb1c1ff 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -71,9 +71,10 @@ builder.Services.AddAuthentication(options => // Queue services builder.Services.AddScoped(); -builder.Services.AddSingleton(); +builder.Services.AddHostedService(); // ✅ GOOD - with proper scope factory // Google Sheets dependencies +Console.WriteLine("Adding Google Sheets dependencies..."); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(provider => diff --git a/tools/http-tests/AutoImport.http b/tools/http-tests/AutoImport.http index a5f2d95..6b35da4 100644 --- a/tools/http-tests/AutoImport.http +++ b/tools/http-tests/AutoImport.http @@ -1,3 +1,3 @@ ### -GET http://localhost:5400/api/Layers/AddPriorityRecords +GET http://localhost:5400/api/Layers/RunQueueJobs/10763478CB738D4ecb2h76g803478CB738D4e From 4fd0b1cd508e1d2b7e258f643bf6af5491816e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sun, 8 Jun 2025 11:08:48 +0200 Subject: [PATCH 11/11] Remove queue --- .../Interfaces/IJobQueueService.cs | 18 - .../Services/JobQueueProcessor.cs | 238 ------------ .../DiunaBI.Core/Services/JobQueueService.cs | 155 -------- .../Controllers/LayersController.cs | 360 +----------------- src/Backend/DiunaBI.WebAPI/Program.cs | 8 +- 5 files changed, 2 insertions(+), 777 deletions(-) delete mode 100644 src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs delete mode 100644 src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs delete mode 100644 src/Backend/DiunaBI.Core/Services/JobQueueService.cs diff --git a/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs b/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs deleted file mode 100644 index 25a70be..0000000 --- a/src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using DiunaBI.Core.Models; - -namespace DiunaBI.Core.Interfaces; - -public interface IJobQueueService -{ - Task EnqueueJobAsync(QueueJob job); - Task DequeueJobAsync(JobType? jobType = null); - Task MarkJobCompletedAsync(Guid jobId); - Task MarkJobFailedAsync(Guid jobId, string error); - Task MarkJobForRetryAsync(Guid jobId, string error); - Task> GetQueueStatusAsync(); - Task GetQueueCountAsync(JobType? jobType = null); - Task GetRunningJobsCountAsync(JobType jobType); -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs b/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs deleted file mode 100644 index 2ea61c5..0000000 --- a/src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs +++ /dev/null @@ -1,238 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Core.Interfaces; -using DiunaBI.Core.Database.Context; -using System; -using System.Threading.Tasks; -using System.Threading; -using System.Net.Http; -using System.Linq; - -namespace DiunaBI.Core.Services; - -public class JobQueueProcessor : BackgroundService -{ - private readonly IServiceScopeFactory _scopeFactory; // ✅ GOOD - używa scope factory - private readonly ILogger _logger; - private readonly ManualResetEventSlim _processSignal = new(false); - - // ❌ USUŃ DIRECT INJECTION scoped services: - // private readonly IJobQueueService _queueService; - // private readonly AppDbContext _db; - - public JobQueueProcessor( - IServiceScopeFactory scopeFactory, // ✅ GOOD - inject scope factory - ILogger logger) - { - _scopeFactory = scopeFactory; - _logger = logger; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("JobQueueProcessor: Started (manual trigger mode)"); - - while (!stoppingToken.IsCancellationRequested) - { - try - { - // Wait for manual trigger - _processSignal.Wait(stoppingToken); - _processSignal.Reset(); - - _logger.LogInformation("JobQueueProcessor: Processing triggered"); - - // ✅ GOOD - create scope for each processing cycle - using var scope = _scopeFactory.CreateScope(); - var queueService = scope.ServiceProvider.GetRequiredService(); - var pluginManager = scope.ServiceProvider.GetRequiredService(); - - await ProcessQueueAsync(queueService, pluginManager, stoppingToken); - } - catch (OperationCanceledException) - { - _logger.LogInformation("JobQueueProcessor: Cancellation requested"); - break; - } - catch (Exception ex) - { - _logger.LogError(ex, "JobQueueProcessor: Error in processing loop"); - await Task.Delay(5000, stoppingToken); // Wait before retry - } - } - - _logger.LogInformation("JobQueueProcessor: Stopped"); - } - - public void TriggerProcessing() - { - _logger.LogInformation("JobQueueProcessor: Manual trigger received"); - _processSignal.Set(); - } - - private async Task ProcessQueueAsync(IJobQueueService queueService, PluginManager pluginManager, CancellationToken cancellationToken) - { - var startTime = DateTime.UtcNow; - var initialQueueSize = await queueService.GetQueueCountAsync(); - int processedJobs = 0; - int failedJobs = 0; - - _logger.LogInformation("JobQueueProcessor: Starting processing of {InitialQueueSize} jobs", initialQueueSize); - - while (!cancellationToken.IsCancellationRequested) - { - // First process all imports (they run sequentially due to Google Sheets API limits) - var importJob = await queueService.DequeueJobAsync(JobType.Import); - if (importJob != null) - { - await ProcessJobAsync(importJob, cancellationToken); - if (importJob.Status == JobStatus.Completed) processedJobs++; - else failedJobs++; - - // Add delay between imports to respect Google Sheets API limits - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); - continue; - } - - // Then process processors (can run in parallel within same priority) - var processJob = await queueService.DequeueJobAsync(JobType.Process); - if (processJob != null) - { - await ProcessJobAsync(processJob, cancellationToken); - if (processJob.Status == JobStatus.Completed) processedJobs++; - else failedJobs++; - continue; - } - - // No more jobs in queue - break; - } - - var endTime = DateTime.UtcNow; - var duration = endTime - startTime; - var finalQueueSize = await queueService.GetQueueCountAsync(); - - _logger.LogInformation("JobQueueProcessor: Processing completed. Duration: {Duration:hh\\:mm\\:ss}, " + - "Initial queue: {InitialQueueSize}, Processed: {ProcessedJobs}, Failed: {FailedJobs}, " + - "Final queue size: {FinalQueueSize}", - duration, initialQueueSize, processedJobs, failedJobs, finalQueueSize); - - if (failedJobs > 0) - { - _logger.LogWarning("JobQueueProcessor: {FailedJobs} jobs failed during processing. Check logs for details.", failedJobs); - } - } - - private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken) - { - try - { - _logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})", - job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority); - - // POPRAWKA: używaj _scopeFactory zamiast _serviceProvider - using var scope = _scopeFactory.CreateScope(); - var dbContext = scope.ServiceProvider.GetRequiredService(); - var pluginManager = scope.ServiceProvider.GetRequiredService(); - var queueService = scope.ServiceProvider.GetRequiredService(); - - // Get the layer with records - var layer = await dbContext.Layers - .Include(x => x.Records) - .FirstOrDefaultAsync(x => x.Id == job.LayerId && !x.IsDeleted, cancellationToken); - - if (layer == null) - { - _logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId); - await queueService.MarkJobFailedAsync(job.Id, "Layer not found"); - job.Status = JobStatus.Failed; - return; - } - - // Process based on job type - switch (job.JobType) - { - case JobType.Import: - var importer = pluginManager.GetImporter(job.PluginName); - if (importer == null) - { - _logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName); - await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found"); - job.Status = JobStatus.Failed; - return; - } - - _logger.LogInformation("JobQueueProcessor: Executing import for layer {LayerName} with plugin {PluginName}", - layer.Name, job.PluginName); - - importer.Import(layer); - break; - - case JobType.Process: - var processor = pluginManager.GetProcessor(job.PluginName); - if (processor == null) - { - _logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName); - await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found"); - job.Status = JobStatus.Failed; - return; - } - - _logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with processor {PluginName}", - layer.Name, job.PluginName); - - processor.Process(layer); - break; - - default: - throw new ArgumentOutOfRangeException(nameof(job.JobType), job.JobType, "Unknown job type"); - } - - await queueService.MarkJobCompletedAsync(job.Id); - job.Status = JobStatus.Completed; - - _logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}", - job.JobType, layer.Name); - } - catch (Exception ex) - { - _logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}", - job.JobType, job.Id, job.LayerName); - - // POPRAWKA: tutaj też używaj _scopeFactory - using var scope = _scopeFactory.CreateScope(); - var queueService = scope.ServiceProvider.GetRequiredService(); - - // Check if it's a retriable error - if (IsRetriableError(ex)) - { - await queueService.MarkJobForRetryAsync(job.Id, ex.Message); - job.Status = JobStatus.Retrying; - } - else - { - await queueService.MarkJobFailedAsync(job.Id, ex.Message); - job.Status = JobStatus.Failed; - } - } - } - - private static bool IsRetriableError(Exception ex) - { - var message = ex.Message.ToLowerInvariant(); - - var retriableErrors = new[] - { - "quota", "rate limit", "timeout", "service unavailable", - "internal server error", "bad gateway", "gateway timeout", - "network", "connection" - }; - - return retriableErrors.Any(error => message.Contains(error)) || - ex is HttpRequestException || - ex is TimeoutException; - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Services/JobQueueService.cs b/src/Backend/DiunaBI.Core/Services/JobQueueService.cs deleted file mode 100644 index c45201c..0000000 --- a/src/Backend/DiunaBI.Core/Services/JobQueueService.cs +++ /dev/null @@ -1,155 +0,0 @@ - -using Microsoft.Extensions.Logging; -using DiunaBI.Core.Models; -using DiunaBI.Core.Interfaces; -using System.Threading.Tasks; -using DiunaBI.Core.Database.Context; -using System; -using System.Linq; -using Microsoft.EntityFrameworkCore; -using System.Collections.Generic; - -namespace DiunaBI.Core.Services; - -public class JobQueueService : IJobQueueService -{ - private readonly AppDbContext _db; - private readonly ILogger _logger; - - public JobQueueService(AppDbContext db, ILogger logger) - { - _db = db; - _logger = logger; - } - - public async Task EnqueueJobAsync(QueueJob job) - { - _db.QueueJobs.Add(job); - await _db.SaveChangesAsync(); - - _logger.LogInformation("JobQueue: Enqueued {JobType} job {JobId} for layer {LayerName} with plugin {PluginName} (priority {Priority})", - job.JobType, job.Id, job.LayerName, job.PluginName, job.Priority); - } - - public async Task DequeueJobAsync(JobType? jobType = null) - { - var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending); - - if (jobType.HasValue) - { - query = query.Where(x => x.JobType == jobType.Value); - } - - var job = await query - .OrderBy(x => x.JobType) // Importers first (0), then Processors (1) - .ThenBy(x => x.Priority) // Then by priority - .ThenBy(x => x.CreatedAt) // Then FIFO - .FirstOrDefaultAsync(); - - if (job != null) - { - job.Status = JobStatus.Running; - job.LastAttemptAt = DateTime.UtcNow; - job.ModifiedAtUtc = DateTime.UtcNow; - await _db.SaveChangesAsync(); - - _logger.LogDebug("JobQueue: Dequeued {JobType} job {JobId} for layer {LayerName} (priority {Priority})", - job.JobType, job.Id, job.LayerName, job.Priority); - } - - return job; - } - - public async Task MarkJobCompletedAsync(Guid jobId) - { - var job = await _db.QueueJobs.FindAsync(jobId); - if (job != null) - { - job.Status = JobStatus.Completed; - job.CompletedAt = DateTime.UtcNow; - job.ModifiedAtUtc = DateTime.UtcNow; - await _db.SaveChangesAsync(); - - _logger.LogInformation("JobQueue: {JobType} job {JobId} completed successfully for layer {LayerName}", - job.JobType, jobId, job.LayerName); - } - } - - public async Task MarkJobFailedAsync(Guid jobId, string error) - { - var job = await _db.QueueJobs.FindAsync(jobId); - if (job != null) - { - job.Status = JobStatus.Failed; - job.LastError = error; - job.ModifiedAtUtc = DateTime.UtcNow; - await _db.SaveChangesAsync(); - - _logger.LogError("JobQueue: {JobType} job {JobId} failed permanently for layer {LayerName}: {Error}", - job.JobType, jobId, job.LayerName, error); - } - } - - public async Task MarkJobForRetryAsync(Guid jobId, string error) - { - var job = await _db.QueueJobs.FindAsync(jobId); - if (job != null) - { - job.RetryCount++; - job.LastError = error; - job.ModifiedAtUtc = DateTime.UtcNow; - - if (job.RetryCount >= job.MaxRetries) - { - await MarkJobFailedAsync(jobId, $"Max retries ({job.MaxRetries}) exceeded. Last error: {error}"); - return; - } - - job.Status = JobStatus.Retrying; - await _db.SaveChangesAsync(); - - // Schedule retry with exponential backoff - var delayMinutes = Math.Pow(2, job.RetryCount); - _ = Task.Delay(TimeSpan.FromMinutes(delayMinutes)) - .ContinueWith(async _ => - { - var retryJob = await _db.QueueJobs.FindAsync(jobId); - if (retryJob?.Status == JobStatus.Retrying) - { - retryJob.Status = JobStatus.Pending; - retryJob.ModifiedAtUtc = DateTime.UtcNow; - await _db.SaveChangesAsync(); - - _logger.LogWarning("JobQueue: {JobType} job {JobId} re-queued for retry {RetryCount}/{MaxRetries} for layer {LayerName}", - retryJob.JobType, jobId, retryJob.RetryCount, retryJob.MaxRetries, retryJob.LayerName); - } - }); - } - } - - public async Task> GetQueueStatusAsync() - { - return await _db.QueueJobs - .OrderBy(x => x.JobType) - .ThenBy(x => x.Priority) - .ThenBy(x => x.CreatedAt) - .ToListAsync(); - } - - public async Task GetQueueCountAsync(JobType? jobType = null) - { - var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending); - - if (jobType.HasValue) - { - query = query.Where(x => x.JobType == jobType.Value); - } - - return await query.CountAsync(); - } - - public async Task GetRunningJobsCountAsync(JobType jobType) - { - return await _db.QueueJobs.CountAsync(x => x.Status == JobStatus.Running && x.JobType == jobType); - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 2fca29c..af9bc21 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -21,7 +21,6 @@ public class LayersController : Controller private readonly IConfiguration _configuration; private readonly PluginManager _pluginManager; private readonly ILogger _logger; - private readonly IJobQueueService _queueService; public LayersController( AppDbContext db, @@ -29,8 +28,7 @@ public class LayersController : Controller GoogleDriveHelper googleDriveHelper, IConfiguration configuration, PluginManager pluginManager, - ILogger logger, - IJobQueueService queueService + ILogger logger ) { _db = db; @@ -39,7 +37,6 @@ public class LayersController : Controller _configuration = configuration; _pluginManager = pluginManager; _logger = logger; - _queueService = queueService; } [HttpGet] @@ -241,51 +238,6 @@ public class LayersController : Controller } } - [HttpGet] - [Route("AutoImportWithQueue/{apiKey}")] - [AllowAnonymous] - public IActionResult AutoImportWithQueue(string apiKey) - { - if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) - { - _logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey); - return Unauthorized(); - } - - var importWorkerLayers = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") - ) - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - - if (importWorkerLayers.Count == 0) - { - _logger.LogInformation("AutoImportQueue: No layers to import"); - return Ok(); - } - - _logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count); - - foreach (var importWorker in importWorkerLayers) - { - try - { - // Queue job implementation would go here - _logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); - } - catch (Exception e) - { - _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", - importWorker.Name, importWorker.Id); - } - } - return Ok(); - } - [HttpGet] [Route("ProcessQueue/{apiKey}")] [AllowAnonymous] @@ -679,42 +631,6 @@ public class LayersController : Controller [HttpGet] [Route("CheckProcessors")] [AllowAnonymous] - public IActionResult CheckProcessors() - { - // get - - /* - var importWorkerLayers = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3") - ) - .OrderByDescending(x => x.CreatedAt) - .AsNoTracking() - .ToList(); - - foreach (var importWorker in importWorkerLayers) - { - var record = new Record - { - Id = Guid.NewGuid(), - LayerId = importWorker.Id, - Code = "Plugin", - Desc1 = "Morska.Import.D3", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") - }; - //_db.Records.Add(record); - } - - //_db.SaveChanges(); - */ - return Ok(); - } - private static void WriteToConsole(params string[] messages) { @@ -809,278 +725,4 @@ public class LayersController : Controller throw; } } - - [HttpGet] - [Route("EnqueueImportWorkers/{apiKey}")] - [AllowAnonymous] - public async Task EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null) - { - if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) - { - _logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); - return Unauthorized(); - } - - try - { - var query = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && - !x.IsDeleted && !x.IsCancelled - ).Take(5); - - // If specific layerId is provided, filter to that layer only - if (layerId.HasValue) - { - query = query.Where(x => x.Id == layerId.Value); - } - - var importWorkerLayers = await query - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToListAsync(); - - _logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}", - importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); - - if (importWorkerLayers.Count == 0) - { - return Ok(new - { - Message = "No import workers found to queue", - QueuedJobs = 0, - TotalQueueSize = await _queueService.GetQueueCountAsync(), - SkippedLayers = 0 - }); - } - - int queuedCount = 0; - - foreach (var importWorker in importWorkerLayers) - { - var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; - if (string.IsNullOrEmpty(pluginName)) - { - _logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", - importWorker.Name, importWorker.Id); - continue; - } - - // Check if plugin exists - var importer = _pluginManager.GetImporter(pluginName); - if (importer == null) - { - _logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping", - pluginName, importWorker.Name, importWorker.Id); - continue; - } - - var job = new QueueJob - { - LayerId = importWorker.Id, - LayerName = importWorker.Name ?? "Unknown", - PluginName = pluginName, - JobType = JobType.Import, - Priority = 0, // All imports have same priority - MaxRetries = 5, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") - }; - - await _queueService.EnqueueJobAsync(job); - queuedCount++; - - _logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}", - importWorker.Name, importWorker.Id, pluginName); - } - - var totalQueueSize = await _queueService.GetQueueCountAsync(); - - _logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}", - queuedCount, totalQueueSize); - - return Ok(new - { - Message = $"Queued {queuedCount} import jobs", - QueuedJobs = queuedCount, - TotalQueueSize = totalQueueSize, - SkippedLayers = importWorkerLayers.Count - queuedCount - }); - } - catch (Exception e) - { - _logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers"); - return BadRequest(e.ToString()); - } - } - - [HttpGet] - [Route("EnqueueProcessWorkers/{apiKey}")] - [AllowAnonymous] - public async Task EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null) - { - if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) - { - _logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey); - return Unauthorized(); - } - - try - { - var query = _db.Layers - .Include(x => x.Records) - .Where(x => - x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && - x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && - !x.IsDeleted && !x.IsCancelled - ).Take(5); - - // If specific layerId is provided, filter to that layer only - if (layerId.HasValue) - { - query = query.Where(x => x.Id == layerId.Value); - } - - var processWorkerLayers = await query - .OrderBy(x => x.CreatedAt) - .AsNoTracking() - .ToListAsync(); - - _logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}", - processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : ""); - - if (processWorkerLayers.Count == 0) - { - return Ok(new - { - Message = "No process workers found to queue", - QueuedJobs = 0, - TotalQueueSize = await _queueService.GetQueueCountAsync(), - SkippedLayers = 0 - }); - } - - int queuedCount = 0; - - foreach (var processWorker in processWorkerLayers) - { - // POPRAWIONE: Używaj Plugin zamiast ProcessorType - var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1; - if (string.IsNullOrEmpty(pluginName)) - { - _logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping", - processWorker.Name, processWorker.Id); - continue; - } - - // POPRAWIONE: Sprawdź czy processor istnieje przez pluginName - var processor = _pluginManager.GetProcessor(pluginName); - if (processor == null) - { - _logger.LogWarning("EnqueueProcessWorkers: Processor {PluginName} not found for layer {LayerName} ({LayerId}), skipping", - pluginName, processWorker.Name, processWorker.Id); - continue; - } - - // Get priority from ProcessWorker record, default to 10 if not found - var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1; - var priority = 10; // Default priority - if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority)) - { - priority = parsedPriority; - } - - var job = new QueueJob - { - LayerId = processWorker.Id, - LayerName = processWorker.Name ?? "Unknown", - PluginName = pluginName, // POPRAWIONE: Używaj pluginName bezpośrednio - JobType = JobType.Process, - Priority = priority, - MaxRetries = 3, - CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), - ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D") - }; - - await _queueService.EnqueueJobAsync(job); - queuedCount++; - - _logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with plugin {PluginName}, priority {Priority}", - processWorker.Name, processWorker.Id, pluginName, priority); - } - - var totalQueueSize = await _queueService.GetQueueCountAsync(); - - _logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}", - queuedCount, totalQueueSize); - - return Ok(new - { - Message = $"Queued {queuedCount} process jobs", - QueuedJobs = queuedCount, - TotalQueueSize = totalQueueSize, - SkippedLayers = processWorkerLayers.Count - queuedCount - }); - } - catch (Exception e) - { - _logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers"); - return BadRequest(e.ToString()); - } - } - - [HttpGet] - [Route("RunQueueJobs/{apiKey}")] - [AllowAnonymous] - public async Task RunQueueJobs(string apiKey) - { - if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) - { - _logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey); - return Unauthorized(); - } - - try - { - var queueSize = await _queueService.GetQueueCountAsync(); - - if (queueSize == 0) - { - return Ok(new - { - Message = "Queue is empty", - QueueSize = 0, - Status = "No jobs to process" - }); - } - - _logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize); - - // ZMIEŃ NA DOSTĘP PRZEZ IHostedService: - var hostedServices = HttpContext.RequestServices.GetServices(); - var queueProcessor = hostedServices.OfType().FirstOrDefault(); - - if (queueProcessor == null) - { - _logger.LogError("RunQueueJobs: JobQueueProcessor not found"); - return BadRequest("JobQueueProcessor not found"); - } - - queueProcessor.TriggerProcessing(); - - return Ok(new - { - Message = $"Queue processing triggered for {queueSize} jobs", - QueueSize = queueSize, - Status = "Processing started in background" - }); - } - catch (Exception e) - { - _logger.LogError(e, "RunQueueJobs: Error triggering queue processing"); - return BadRequest(e.ToString()); - } - } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index bb1c1ff..440e6c5 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -69,10 +69,6 @@ builder.Services.AddAuthentication(options => }; }); -// Queue services -builder.Services.AddScoped(); -builder.Services.AddHostedService(); // ✅ GOOD - with proper scope factory - // Google Sheets dependencies Console.WriteLine("Adding Google Sheets dependencies..."); builder.Services.AddSingleton(); @@ -142,9 +138,7 @@ app.Use(async (context, next) => if (token.Length > 0 && !context.Request.Path.ToString().Contains("getForPowerBI") && !context.Request.Path.ToString().Contains("getConfiguration") - && !context.Request.Path.ToString().Contains("DataInbox/Add") - && !context.Request.Path.ToString().Contains("AddPluginName") // TODO: Remove this - && !context.Request.Path.ToString().Contains("GetImportWorkers")) + && !context.Request.Path.ToString().Contains("DataInbox/Add")) { var handler = new JwtSecurityTokenHandler(); var data = handler.ReadJwtToken(token.Split(' ')[1]);