From d05dc34e97fadf8deef3d1d767c8335a1c983ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zieli=C5=84ski?= Date: Sat, 7 Jun 2025 13:51:27 +0200 Subject: [PATCH] Refactor processors --- .../Importers/MorskaD1Importer.cs | 61 +- .../Importers/MorskaFK2Importer.cs | 61 +- ...aImporter.cs => MorskaStandardImporter.cs} | 70 +- .../Processors/T1R1Processor.cs | 544 ++++++++++----- .../Processors/T1R3Processor.cs | 452 ++++++++++--- ...T3MultiSourceCopySelectedCodesProcessor.cs | 273 ++++++-- ...ceCopySelectedCodesYearSummaryProcessor.cs | 200 +++++- .../T3MultiSourceSummaryProcessor.cs | 290 ++++++-- .../T3MultiSourceYearSummaryProcessor.cs | 312 ++++++--- .../Processors/T3SingleSourceProcessor.cs | 297 ++++++-- .../T3SourceYearSummaryProcessor.cs | 213 +++++- .../Processors/T4R2Processor.cs | 634 ++++++++++++------ .../Processors/T4SingleSourceProcessor.cs | 224 ++++++- .../Processors/T5LastValuesProcessor.cs | 282 ++++++-- 14 files changed, 2996 insertions(+), 917 deletions(-) rename src/Backend/DiunaBI.Plugins.Morska/Importers/{MorskaImporter.cs => MorskaStandardImporter.cs} (86%) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs index 52d3975..aec4c02 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs @@ -26,6 +26,10 @@ public class MorskaD1Importer : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaD1Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -43,6 +47,10 @@ public class MorskaD1Importer : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + LoadConfiguration(importWorker); if (!ShouldPerformImport(importWorker)) @@ -65,6 +73,46 @@ public class MorskaD1Importer : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -164,8 +212,8 @@ public class MorskaD1Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); if (data == null || data.Count < 2) { @@ -239,10 +287,10 @@ public class MorskaD1Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); var newRecords = (from t in data @@ -279,8 +327,7 @@ public class MorskaD1Importer : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs index d8dd909..e4ccda7 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs @@ -26,6 +26,10 @@ public class MorskaFk2Importer : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } + // Cache for sheet data + private IList>? _cachedSheetData; + private string? _cachedDataKey; + public MorskaFk2Importer( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -43,6 +47,10 @@ public class MorskaFk2Importer : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // ✅ Clear cache at start + _cachedSheetData = null; + _cachedDataKey = null; + LoadConfiguration(importWorker); if (!ShouldPerformImport(importWorker)) @@ -65,6 +73,46 @@ public class MorskaFk2Importer : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // ✅ Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + // ✅ Dodaj metodę cache + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -164,8 +212,8 @@ public class MorskaFk2Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); if (data == null || data.Count == 0) { @@ -256,10 +304,10 @@ public class MorskaFk2Importer : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + // ✅ Użyj cache zamiast bezpośredniego API + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); if (data != null) @@ -315,8 +363,7 @@ public class MorskaFk2Importer : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs similarity index 86% rename from src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs rename to src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs index 15cc887..49a9b5e 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaImporter.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs @@ -7,13 +7,13 @@ using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Importers; -public class MorskaImporter : MorskaBaseImporter +public class MorskaStandardImporter : MorskaBaseImporter { public override string ImporterType => "Morska.Import.Standard"; private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; - private readonly ILogger _logger; + private readonly ILogger _logger; // Configuration properties private string? SheetId { get; set; } @@ -26,10 +26,13 @@ public class MorskaImporter : MorskaBaseImporter private DateTime? EndDate { get; set; } private bool IsEnabled { get; set; } - public MorskaImporter( + private IList>? _cachedSheetData; + private string? _cachedDataKey; + + public MorskaStandardImporter( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, - ILogger logger) + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; @@ -43,6 +46,10 @@ public class MorskaImporter : MorskaBaseImporter _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", ImporterType, importWorker.Name, importWorker.Id); + // Clear cache before import + _cachedSheetData = null; + _cachedDataKey = null; + // Load configuration from layer records LoadConfiguration(importWorker); @@ -69,6 +76,45 @@ public class MorskaImporter : MorskaBaseImporter ImporterType, importWorker.Name, importWorker.Id); throw; } + finally + { + // Clear cache after import + _cachedSheetData = null; + _cachedDataKey = null; + } + } + + private IList>? GetSheetData() + { + var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}"; + + if (_cachedSheetData != null && _cachedDataKey == currentDataKey) + { + _logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}", + ImporterType, currentDataKey); + return _cachedSheetData; + } + + try + { + _logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}", + ImporterType, currentDataKey); + + var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); + _cachedSheetData = dataRangeResponse.Values; + _cachedDataKey = currentDataKey; + + _logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet", + ImporterType, _cachedSheetData?.Count ?? 0); + + return _cachedSheetData; + } + catch (Exception e) + { + _logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}", + ImporterType, SheetId); + throw; + } } private void LoadConfiguration(Layer importWorker) @@ -171,14 +217,13 @@ public class MorskaImporter : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + var data = GetSheetData(); if (data == null || data.Count < 2) { _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", ImporterType, importWorker.Name); - return true; // Assume up to date if no data + return true; } // Check if the number of columns matches @@ -199,7 +244,6 @@ public class MorskaImporter : MorskaBaseImporter continue; } - // Check if the record exists in the database - add null check var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); if (existingRecord == null || existingRecord.Value1 != value) { @@ -243,12 +287,9 @@ public class MorskaImporter : MorskaBaseImporter try { - var dataRangeResponse = _googleSheetValues.Get( - SheetId!, - $"{SheetTabName}!{DataRange}").Execute(); - var data = dataRangeResponse.Values; + var data = GetSheetData(); - _logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", + _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache", ImporterType, data?.Count ?? 0); if (data != null && data.Count >= 2) @@ -284,8 +325,7 @@ public class MorskaImporter : MorskaBaseImporter } catch (Exception e) { - _logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", - ImporterType, SheetId); + _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType); throw; } } diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index aa66b0c..d5801e6 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -18,6 +18,12 @@ public class T1R1Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + private string? GoogleSheetName { get; set; } + public T1R1Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,162 +33,321 @@ public class T1R1Processor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + // Load Google Sheet name + GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(GoogleSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for each month + var newRecords = ProcessAllMonths(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) - .ToList(); + return processedLayer; + } + private List ProcessAllMonths() + { var newRecords = new List(); for (var month = 1; month < 14; month++) { - if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))) + // Skip future months (except month 13 which is summary) + if (Year > DateTime.UtcNow.Year || + (Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)) { + _logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}", + ProcessorType, Year, month); continue; } - var records = new List(); - foreach (var source in sources) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3") - ).Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); + _logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}", + ProcessorType, month, Year); - if (dataSource == null) - { - throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found."); - } + var monthRecords = ProcessSingleMonth(month); + newRecords.AddRange(monthRecords); - var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault(); - if (codesRecord != null) - { - var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); - records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!)))); - } - else - { - records.AddRange(dataSource.Records!); - } - } - - if (dynamicCodes != null) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - records.Add(calc.CalculateT1(records)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) - { - _logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - } - - newRecords.AddRange(records.Select(x => new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - } - )); + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}", + ProcessorType, monthRecords.Count, month); } - if (isNew) + return newRecords; + } + + private List ProcessSingleMonth(int month) + { + var records = new List(); + + // Collect records from all sources + foreach (var source in Sources!) + { + var sourceRecords = GetSourceRecords(source, month); + records.AddRange(sourceRecords); + } + + // Process dynamic codes (calculations) + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(records, month); + records.AddRange(calculatedRecords); + } + + // Create final records with month suffix + var monthRecords = records.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + return monthRecords; + } + + private List GetSourceRecords(Record source, int month) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found"); + } + + var sourceRecords = new List(); + + // Check if there are specific codes configured for this source + var codesRecord = Sources! + .Where(x => x.Code == $"Codes-{source.Desc1}") + .FirstOrDefault(); + + if (codesRecord != null) + { + var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); + sourceRecords.AddRange(dataSource.Records! + .Where(x => codes.Contains(int.Parse(x.Code!)))); + + _logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes", + ProcessorType, source.Desc1, codes.Count); + } + else + { + sourceRecords.AddRange(dataSource.Records!); + + _logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records", + ProcessorType, source.Desc1, dataSource.Records!.Count); + } + + return sourceRecords; + } + + private List ProcessDynamicCodes(List records, int month) + { + var calculatedRecords = new List(); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) + { + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}", + ProcessorType, dynamicCode.Id, month); + continue; + } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}", + ProcessorType, calc.Expression, dynamicCode.Id, month); + continue; + } + + var calculatedRecord = calc.CalculateT1(records); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}", + ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}", + ProcessorType, dynamicCode.Desc1, month); + } + } + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (sheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - UpdateReport(processedLayer.Id, sheetName); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -193,92 +358,139 @@ public class T1R1Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, string sheetName) + private void UpdateGoogleSheetReport(Guid sourceId) { - const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; - var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4"); - var response = request.Execute(); + try + { + _logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); - var r1 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); + const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; + + // Get processed layer data + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); - var codesRow = response.Values[0]; + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + // Get codes from sheet header + var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute(); + var codesRow = codesResponse.Values[0]; + + // Update monthly data (months 1-12) + UpdateMonthlyData(sheetId, codesRow, processedLayer); + + // Update summary row (month 13) + UpdateSummaryData(sheetId, codesRow, processedLayer); + + // Update timestamps + UpdateTimestamps(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}", + ProcessorType, GoogleSheetName); + throw; + } + } + + private void UpdateMonthlyData(string sheetId, IList codesRow, Layer processedLayer) + { var valueRange = new ValueRange { Values = new List>() }; - for (var i = 1; i <= 12; i++) + // Process months 1-12 + for (var month = 1; month <= 12; month++) { - var values = new List(); + var monthValues = new List(); foreach (string code in codesRow) { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}"); + monthValues.Add(record?.Value1?.ToString() ?? "0"); } - valueRange.Values.Add(values); + valueRange.Values.Add(monthValues); } - // sum - var valuesSum = new List(); - var emptyRow = new List(); - foreach (string code in codesRow) - { - var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); - emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } - } - - valueRange.Values.Add(emptyRow); - valueRange.Values.Add(valuesSum); - - var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20"); - update.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); - // update time - var timeUtc = new List - { - r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange + _logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType); + } + + private void UpdateSummaryData(string sheetId, IList codesRow, Layer processedLayer) + { + var valueRange = new ValueRange { Values = new List>() }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); + // Empty row + var emptyRow = new List(); + for (int i = 0; i < codesRow.Count; i++) + { + emptyRow.Add(""); + } + valueRange.Values.Add(emptyRow); + + // Summary row (month 13) + var summaryValues = new List(); + foreach (string code in codesRow) + { + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + summaryValues.Add(record?.Value1?.ToString() ?? "0"); + } + valueRange.Values.Add(summaryValues); + + var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs index 7549f5a..7a4cc34 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs @@ -18,6 +18,10 @@ public class T1R3Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + public T1R3Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,46 +31,140 @@ public class T1R3Processor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1; - if (source == null) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheet report + UpdateGoogleSheetReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - - var newRecords = new List(); - - string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$"; + private List GetDataSources() + { + string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$"; + var dataSources = _db.Layers .Where(x => !x.IsDeleted && !x.IsCancelled) .Include(layer => layer.Records!) @@ -75,59 +173,127 @@ public class T1R3Processor : MorskaBaseProcessor .Where(x => Regex.IsMatch(x.Name!, pattern)) .ToList(); + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"No data sources found for pattern: {pattern}"); + } + + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}", + ProcessorType, dataSources.Count, pattern); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); + foreach (var dataSource in dataSources) { - var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); - if (month == null) + var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); + if (monthStr == null || !int.TryParse(monthStr, out var month)) { - throw new Exception($"Month not found: {dataSource.Name}"); + _logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}", + ProcessorType, dataSource.Name); + continue; } - foreach (var record in dataSource.Records!) + _logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}", + ProcessorType, dataSource.Name, month); + + var sourceRecords = ProcessDataSourceRecords(dataSource, month); + newRecords.AddRange(sourceRecords); + + _logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}", + ProcessorType, sourceRecords.Count, dataSource.Name); + } + + _logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDataSourceRecords(Layer dataSource, int month) + { + var newRecords = new List(); + + foreach (var record in dataSource.Records!) + { + if (record.Value1 == null) { - if (record.Value1 == null) continue; - for (var i = 1; i < 33; i++) + _logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null", + ProcessorType, record.Code); + continue; + } + + // Process values for positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(record, i); + if (value == null) { - if (ProcessHelper.GetValue(record, i) == null) continue; - - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{record.Code}{month}{i:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = i == 1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100, - Desc1 = record.Desc1 - }; - - newRecords.Add(newRecord); + continue; } + + var baseValue = (double)record.Value1!; + var positionValue = (double)value; + var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100; + + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = $"{record.Code}{month:D2}{i:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = calculatedValue, + Desc1 = record.Desc1 + }; + + newRecords.Add(newRecord); + + _logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}", + ProcessorType, newRecord.Code, newRecord.Value1, record.Code); } } - if (isNew) + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - UpdateReport(processedLayer.Id, year); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -138,90 +304,164 @@ public class T1R3Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, int year) + private void UpdateGoogleSheetReport(Guid sourceId) { - const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; - - var r3 = _db.Layers - .Where(x => x.Id == sourceId) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - - for (var i = 1; i <= 12; i++) + try { - var sheetName = ProcessHelper.GetSheetName(i, year); + _logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}", + ProcessorType, sourceId); + + const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; + + var processedLayer = _db.Layers + .Where(x => x.Id == sourceId) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + // Update sheets for all months + for (var month = 1; month <= 12; month++) + { + UpdateMonthSheet(sheetId, processedLayer, month); + } + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month) + { + var sheetName = ProcessHelper.GetSheetName(month, Year); + + try + { + _logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + + // Get codes from sheet ValueRange? dataRangeResponse; try { dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); } - catch + catch (Exception e) { - continue; // Sheet not exist + _logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}", + ProcessorType, sheetName, e.Message); + return; } - if (dataRangeResponse == null) continue; // Sheet not exist - var data = dataRangeResponse.Values; - - var updateValueRange = new ValueRange + if (dataRangeResponse?.Values == null) { - Values = new List>() - }; - - foreach (var row in data) - { - if (row.Count == 0) continue; - var code = row[0].ToString(); - - var updateRow = new List(); - - for (var j = 1; j < 16; j++) - { - var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}"); - if (codeRecord is { Value1: not null }) - { - updateRow.Add(codeRecord.Value1); - } - else - { - updateRow.Add(""); - } - } - - updateValueRange.Values.Add(updateRow); + _logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping", + ProcessorType, sheetName); + return; } - dataRangeResponse.Values = data; - var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); - update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - update.Execute(); + // Update data + UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month); - // update time - var timeUtc = new List - { - r3!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r3.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; - var valueRangeTime = new ValueRange - { - Values = new List>() - }; - valueRangeTime.Values.Add(timeUtc); - valueRangeTime.Values.Add(timeWarsaw); + // Update timestamps + UpdateSheetTimestamps(sheetId, sheetName, processedLayer); - var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); - updateTimeUtc.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; - updateTimeUtc.Execute(); + _logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}", + ProcessorType, sheetName); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}", + ProcessorType, sheetName, month); + throw; } } + + private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList> codeRows, int month) + { + var updateValueRange = new ValueRange + { + Values = new List>() + }; + + foreach (var row in codeRows) + { + if (row.Count == 0) continue; + + var code = row[0].ToString(); + var updateRow = new List(); + + // Process columns C to Q (positions 1-15) + for (var position = 1; position <= 15; position++) + { + var recordCode = $"{code}{month:D2}{position:D2}"; + var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode); + + if (codeRecord?.Value1 != null) + { + updateRow.Add(codeRecord.Value1); + _logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}", + ProcessorType, codeRecord.Value1, recordCode); + } + else + { + updateRow.Add(""); + } + } + + updateValueRange.Values.Add(updateRow); + } + + // Update sheet with new values + var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); + update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + update.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}", + ProcessorType, updateValueRange.Values.Count, sheetName); + } + + private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer) + { + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); + + var valueRangeTime = new ValueRange + { + Values = new List> + { + new List { timeUtc }, + new List { timeWarsaw } + } + }; + + var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); + updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + updateTime.Execute(); + + _logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, sheetName, timeUtc, timeWarsaw); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs index 277b0d4..11fce8d 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs @@ -13,6 +13,13 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private string? Codes { get; set; } + private List? CodesList { get; set; } + public T3MultiSourceCopySelectedCodesProcessor( AppDbContext db, ILogger logger) @@ -20,102 +27,268 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); } - var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; - if (codes == null) + catch (Exception e) { - throw new Exception("Codes record not found"); + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var codesList = ProcessHelper.ParseCodes(codes); + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load codes + Codes = GetRecordValue(processWorker.Records, "Codes"); + if (string.IsNullOrEmpty(Codes)) + { + throw new InvalidOperationException("Codes record not found"); + } + + // Parse codes list + CodesList = ProcessHelper.ParseCodes(Codes); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}", + ProcessorType, Year, Month, Sources.Count, CodesList.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } + + private List GetDataSources() + { + var dataSources = new List(); + + foreach (var source in Sources!) + { + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSource == null) + { + _logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3", + ProcessorType, Year, Month, source.Desc1); + continue; + } + + dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + } - var dataSources = sources.Select(source => _db.Layers - .Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) - .Include(x => x.Records).AsNoTracking() - .FirstOrDefault()) - .OfType() - .ToList(); if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources", + ProcessorType, dataSources.Count); - var newRecords = dataSources - .SelectMany(x => x.Records!) - .Where(x => codesList.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = x.Code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - for (var i = 1; i < 33; i++) - { - ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); - } - return newRecord; - }) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList(); + + var filteredRecords = allSourceRecords + .Where(x => !string.IsNullOrEmpty(x.Code) && + int.TryParse(x.Code, out var code) && + CodesList!.Contains(code)) .ToList(); - if (isNew) + + _logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes", + ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count); + + var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateCopiedRecord(Record sourceRecord) + { + var newRecord = new Record + { + Id = Guid.NewGuid(), + Code = sourceRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(sourceRecord, i); + ProcessHelper.SetValue(newRecord, i, value); + } + + _logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32", + ProcessorType, newRecord.Code); + + return newRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -126,6 +299,12 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs index 92f5089..dd7f50c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs @@ -13,6 +13,9 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + public T3MultiSourceCopySelectedCodesYearSummaryProcessor( AppDbContext db, ILogger logger) @@ -20,67 +23,176 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}", + ProcessorType, Year); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}", + ProcessorType, Year); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = new List(); - for (var i = 1; i < 13; i++) + for (var month = 1; month <= 12; month++) { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-AB-T3")) + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + if (dataSource != null) { dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); } } if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}"); } - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); - foreach (var baseRecord in dataSources.Last().Records!) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -88,34 +200,60 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - for (var i = 1; i < 33; i++) + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0); + ProcessHelper.SetValue(processedRecord, position, totalValue); } newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -126,6 +264,12 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index aebac1b..e7f642d 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -14,6 +14,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + public T3MultiSourceSummaryProcessor( AppDbContext db, ILogger logger) @@ -24,47 +30,159 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources", + ProcessorType, Year, Month, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) @@ -73,17 +191,28 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -91,82 +220,105 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; + + // Sum values from all sources for positions 1-32 for (var i = 1; i < 33; i++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); } - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) { - foreach (var dynamicCode in dynamicCodes) + try { - try + if (string.IsNullOrEmpty(dynamicCode.Desc1)) { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; } - catch (Exception e) + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) { - _logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); } } - if (isNew) + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -177,6 +329,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index b418e7e..75b8f43 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -14,6 +14,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private List? DynamicCodes { get; set; } + public T3MultiSourceYearSummaryProcessor( AppDbContext db, ILogger logger) @@ -24,69 +29,180 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); + } + + // Load dynamic codes + DynamicCodes = processWorker.Records + .Where(x => x.Code!.Contains("DynamicCode-")) + .OrderBy(x => int.Parse(x.Code!.Split('-')[1])) + .ToList(); + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}", + ProcessorType, Year, Sources.Count, DynamicCodes.Count); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records (sum by base codes with validation) + var newRecords = ProcessRecords(dataSources); + + // Process dynamic codes if configured + if (DynamicCodes != null && DynamicCodes.Count > 0) + { + var calculatedRecords = ProcessDynamicCodes(newRecords); + newRecords.AddRange(calculatedRecords); + } + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } - var newRecords = new List(); - - var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")) + private List GetDataSources() + { + var dataSources = Sources!.Select(source => + _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault()) .OfType() .ToList(); - + if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}"); } - var allRecords = dataSources - .SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}", + ProcessorType, dataSources.Count, Year); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes", + ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count); foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); - var codeRecordsValidation = allRecords.Where(x => - x.Code![1..] == baseCode) - .ToList(); + var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -94,95 +210,119 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; + + // Validation record for double-checking calculations var validationRecord = new Record(); + + // Sum values from all sources for positions 1-32 with validation for (var i = 1; i < 33; i++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(processedRecord, i, totalValue); - ProcessHelper.SetValue(validationRecord, i, - codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); + // Validation calculation (identical to main calculation) + var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i)); + ProcessHelper.SetValue(validationRecord, i, validationValue); - if ( - double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - - ProcessHelper.GetValue(validationRecord, i))!) > 0.01) + // Validate that both calculations match + var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!); + if (difference > 0.01) { - throw new Exception($"ValidationError: Code {baseCode}, " + - $"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " + - $"{ProcessHelper.GetValue(validationRecord, i)})"); + throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})"); } } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records", + ProcessorType, baseCode, codeRecords.Count); } - // Dynamic Codes - var dynamicCodes = processWorker.Records? - .Where(x => x.Code!.Contains("DynamicCode-")) - .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); - - if (dynamicCodes != null && dynamicCodes.Count != 0) - { - foreach (var dynamicCode in dynamicCodes) - { - try - { - if (dynamicCode.Desc1 == null) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } - - var calc = new BaseCalc(dynamicCode.Desc1); - if (!calc.IsFormulaCorrect()) - { - _logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - continue; - } + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); - try - { - newRecords.Add(calc.CalculateT3(newRecords)); - } - catch (Exception e) - { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", - calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); - } - } - catch (Exception e) + return newRecords; + } + + private List ProcessDynamicCodes(List baseRecords) + { + var calculatedRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes", + ProcessorType, DynamicCodes!.Count); + + foreach (var dynamicCode in DynamicCodes!) + { + try + { + if (string.IsNullOrEmpty(dynamicCode.Desc1)) { - _logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", - dynamicCode.Id, processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing", + ProcessorType, dynamicCode.Id); + continue; } + + var calc = new BaseCalc(dynamicCode.Desc1); + if (!calc.IsFormulaCorrect()) + { + _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct", + ProcessorType, calc.Expression, dynamicCode.Id); + continue; + } + + var calculatedRecord = calc.CalculateT3(baseRecords); + calculatedRecords.Add(calculatedRecord); + + _logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}", + ProcessorType, calculatedRecord.Code, calculatedRecord.Value1); + } + catch (Exception e) + { + _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error", + ProcessorType, dynamicCode.Desc1); } } - if (isNew) + _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records", + ProcessorType, calculatedRecords.Count); + + return calculatedRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } - + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -193,6 +333,12 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs index 138c89a..57c0fa6 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs @@ -15,6 +15,13 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayerName { get; set; } + private string? Source { get; set; } + private Layer? SourceImportWorker { get; set; } + public T3SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -24,70 +31,184 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor _googleSheetValues = googleSheetValues; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) + try { - throw new Exception("SourceLayer record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); } - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) + catch (Exception e) { - throw new Exception("SourceImportWorkerL layer not found"); + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; } - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) { - throw new Exception("Source record not found"); + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year and month + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer name + SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayerName)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source name + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayerName, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + // Find source import worker + SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled); + if (SourceImportWorker == null) + { + errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found"); + } + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(); + + // Process records + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = _db.Layers .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled) .OrderBy(x => x.CreatedAt) .AsNoTracking() .ToList(); + if (dataSources.Count == 0) { - throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); + throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'"); } + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing", + ProcessorType, dataSources.Count); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var newRecords = new List(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records", + ProcessorType, allRecords.Count); + foreach (var baseRecord in dataSources.Last().Records!) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); @@ -100,67 +221,104 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor ModifiedAt = DateTime.UtcNow }; - var lastDayInMonth = DateTime.DaysInMonth(year, month); - //day 1 - var firstVal = codeRecords - .Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0; - ProcessHelper.SetValue(processedRecord, 1, firstVal); - var previousValue = firstVal; - //days 2-29/30 - for (var i = 2; i < lastDayInMonth; i++) - { - var dayVal = codeRecords - .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1; - if (dayVal == null) - { - ProcessHelper.SetValue(processedRecord, i, 0); - } - else - { - var processedVal = dayVal - previousValue; - ProcessHelper.SetValue(processedRecord, i, processedVal); - previousValue = (double)dayVal; - } - } - //last day - var lastVal = codeRecords - .Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1; - - if (lastVal == null) - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); - } - else - { - ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); - } - - // copy last value - var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; - ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + ProcessDailyValues(processedRecord, codeRecords); newRecords.Add(processedRecord); } + _logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void ProcessDailyValues(Record processedRecord, List codeRecords) + { + var lastDayInMonth = DateTime.DaysInMonth(Year, Month); + + // Day 1 - first value for the month + var firstVal = codeRecords + .Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1)) + .MaxBy(x => x.CreatedAt)?.Value1 ?? 0; + + ProcessHelper.SetValue(processedRecord, 1, firstVal); + var previousValue = firstVal; + + // Days 2 to last-1 - daily differences + for (var i = 2; i < lastDayInMonth; i++) + { + var dayVal = codeRecords + .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (dayVal == null) + { + ProcessHelper.SetValue(processedRecord, i, 0); + } + else + { + var processedVal = dayVal - previousValue; + ProcessHelper.SetValue(processedRecord, i, processedVal); + previousValue = (double)dayVal; + } + } + + // Last day - special handling + var lastVal = codeRecords + .Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth)) + .MaxBy(x => x.CreatedAt)?.Value1; + + if (lastVal == null) + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0); + } + else + { + ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); + } + + // Copy last value to position 32 + var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; + ProcessHelper.SetValue(processedRecord, 32, valueToCopy); + + _logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}", + ProcessorType, processedRecord.Code, valueToCopy); + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id); + if (isNew) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -171,7 +329,12 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 09b6972..0fcd769 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -15,6 +15,10 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private string? Source { get; set; } + public T3SourceYearSummaryProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -27,72 +31,181 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}", + ProcessorType, Year, Source); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources for all months + var dataSources = GetDataSources(); + + // Process records (sum all monthly values) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources() + { var dataSources = new List(); - for (var i = 1; i < 13; i++) + + for (var month = 1; month <= 12; month++) { - var j = i; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed - && !x.IsDeleted && !x.IsCancelled - && x.Name != null && x.Name.Contains($"{year}/{j:D2}-{source}-T3")) + var dataSource = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3")) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + if (dataSource != null) { dataSources.Add(dataSource); + _logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records", + ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0); + } + else + { + _logger.LogDebug("{ProcessorType}: No data source found for month {Month}", + ProcessorType, month); } } if (dataSources.Count == 0) { - throw new Exception("DataSources are empty"); + throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}"); } - var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}", + ProcessorType, dataSources.Count, Year, Source); - foreach (var baseRecord in dataSources.Last().Records!) + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var baseRecords = dataSources.Last().Records!; + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records", + ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count); + + foreach (var baseRecord in baseRecords) { var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); + var processedRecord = new Record { Id = Guid.NewGuid(), @@ -100,38 +213,60 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow }; - for (var i = 1; i < 33; i++) + + // Sum values from all months for positions 1-32 + for (var position = 1; position <= 32; position++) { - ProcessHelper.SetValue(processedRecord, i, - codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); + var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position)); + ProcessHelper.SetValue(processedRecord, position, totalValue); } + newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records", + ProcessorType, baseRecord.Code, codeRecords.Count); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } - + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -142,6 +277,12 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index 6a129f0..e2fccf9 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -7,7 +7,6 @@ using DiunaBI.Core.Models; using DiunaBI.Core.Database.Context; using DiunaBI.Core.Services; - namespace DiunaBI.Plugins.Morska.Processors; public class T4R2Processor : MorskaBaseProcessor @@ -18,6 +17,13 @@ public class T4R2Processor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private List? Sources { get; set; } + private string? LayerName { get; set; } + private string? ReportSheetName { get; set; } + private string? InvoicesSheetName { get; set; } + public T4R2Processor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -30,200 +36,365 @@ public class T4R2Processor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); - if (sources!.Count == 0) + try { - throw new Exception("Source record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; - if (layerName == null) + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) { - throw new Exception("LayerName record not found"); + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load sources + Sources = processWorker.Records.Where(x => x.Code == "Source").ToList(); + if (Sources.Count == 0) + { + throw new InvalidOperationException("Source records not found"); } + // Load layer name + LayerName = GetRecordValue(processWorker.Records, "LayerName"); + if (string.IsNullOrEmpty(LayerName)) + { + throw new InvalidOperationException("LayerName record not found"); + } + + // Load report sheet name + ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName"); + if (string.IsNullOrEmpty(ReportSheetName)) + { + throw new InvalidOperationException("GoogleSheetName record not found"); + } + + // Load invoices sheet name + InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices"); + if (string.IsNullOrEmpty(InvoicesSheetName)) + { + throw new InvalidOperationException("GoogleSheetName-Invoices record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}", + ProcessorType, Year, Sources.Count, LayerName); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Sources == null || Sources.Count == 0) errors.Add("No sources configured"); + if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required"); + if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required"); + if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources", + ProcessorType, Year, Sources!.Count); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Process records for all sources + var newRecords = ProcessSources(); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + // Update Google Sheets reports + UpdateReport(processedLayer.Id); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-{LayerName}"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - + return processedLayer; + } + private List ProcessSources() + { var newRecords = new List(); - foreach (var source in sources) + foreach (var source in Sources!) { - var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") - ?.Desc1; - var sourceCodes = new List(); - if (rawSourceCodes != null) - { - sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); - } + _logger.LogDebug("{ProcessorType}: Processing source {Source}", + ProcessorType, source.Desc1); - List lastSourceCodes = []; + var sourceCodes = GetSourceCodes(source); + var sourceRecords = ProcessSourceData(source, sourceCodes); + newRecords.AddRange(sourceRecords); - for (var month = 1; month <= 12; month++) - { - if ((year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || year < DateTime.UtcNow.Year) - { - var monthCopy = month; - var dataSource = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSource != null) - { - lastSourceCodes = dataSource.Records!.Select(x => x.Code!).ToList(); - var news = dataSource.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, - Desc1 = x.Desc1 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - else - { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id); - } - } - else - { - //0 values for future months - if (source.Desc1 == "FK2" || lastSourceCodes.Count <= 0) continue; - var news = lastSourceCodes - .Where(x => sourceCodes.Contains(int.Parse(x))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x}{month:D2}", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = 0, - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); - } - } + _logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records", + ProcessorType, source.Desc1, sourceRecords.Count); + } - // year summary - var dataSourceSum = _db.Layers.Where(x => - x.Type == LayerType.Processed && - !x.IsDeleted && !x.IsCancelled && - x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") - ) - .Include(x => x.Records) - .AsNoTracking() - .FirstOrDefault(); - if (dataSourceSum != null) + _logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private List GetSourceCodes(Record source) + { + var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}"); + if (string.IsNullOrEmpty(rawSourceCodes)) + { + return new List(); + } + + return ProcessHelper.ParseCodes(rawSourceCodes); + } + + private List ProcessSourceData(Record source, List sourceCodes) + { + var sourceRecords = new List(); + var lastSourceCodes = new List(); + + // Process monthly data (1-12) + for (var month = 1; month <= 12; month++) + { + var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes); + sourceRecords.AddRange(monthRecords); + } + + // Process year summary (month 13) + var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes); + sourceRecords.AddRange(yearSummaryRecords); + + return sourceRecords; + } + + private List ProcessMonthData(Record source, List sourceCodes, int month, List lastSourceCodes) + { + var monthRecords = new List(); + + if (IsDataAvailableForMonth(month)) + { + var dataSource = GetMonthDataSource(source, month); + if (dataSource != null) { - var news = dataSourceSum.Records! - .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) - .Select(x => - { - var newRecord = new Record - { - Id = Guid.NewGuid(), - Code = $"{x.Code}13", - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow, - Value1 = x.Value32 - }; - return newRecord; - } - ).ToList(); - newRecords.AddRange(news); + lastSourceCodes.Clear(); + lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!)); + + var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes); + var processedRecords = CreateMonthRecords(filteredRecords, source, month); + monthRecords.AddRange(processedRecords); + + _logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records", + ProcessorType, month, source.Desc1, processedRecords.Count); } else { - _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", - $"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id); + _logger.LogWarning("{ProcessorType}: Data source {DataSource} not found", + ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T"); + } + } + else + { + // Future months - create zero value records (except for FK2) + if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0) + { + var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month); + monthRecords.AddRange(futureRecords); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}", + ProcessorType, futureRecords.Count, month); } } - if (isNew) + return monthRecords; + } + + private bool IsDataAvailableForMonth(int month) + { + return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year; + } + + private Layer? GetMonthDataSource(Record source, int month) + { + return _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + } + + private List FilterRecords(IEnumerable records, List sourceCodes) + { + return records + .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) + .ToList(); + } + + private List CreateMonthRecords(List filteredRecords, Record source, int month) + { + return filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, + Desc1 = x.Desc1 + }).ToList(); + } + + private List CreateFutureMonthRecords(List lastSourceCodes, List sourceCodes, int month) + { + return lastSourceCodes + .Where(x => sourceCodes.Contains(int.Parse(x))) + .Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x}{month:D2}", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = 0 + }).ToList(); + } + + private List ProcessYearSummaryData(Record source, List sourceCodes) + { + var dataSourceSum = _db.Layers + .Where(x => x.Type == LayerType.Processed && + !x.IsDeleted && !x.IsCancelled && + x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T")) + .Include(x => x.Records) + .AsNoTracking() + .FirstOrDefault(); + + if (dataSourceSum == null) + { + _logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found", + ProcessorType, $"{Year}/13-{source.Desc1}-T3"); + return new List(); + } + + var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes); + var yearSummaryRecords = filteredRecords.Select(x => new Record + { + Id = Guid.NewGuid(), + Code = $"{x.Code}13", + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow, + Value1 = x.Value32 + }).ToList(); + + _logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}", + ProcessorType, yearSummaryRecords.Count, source.Desc1); + + return yearSummaryRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; - if (reportSheetName == null) - { - throw new Exception("GoogleSheetName record not found"); - } - - var invoicesSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName-Invoices")?.Desc1; - if (invoicesSheetName == null) - { - throw new Exception("GoogleSheetName-Invoices record not found"); - } - - UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName); - - _logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -234,153 +405,196 @@ public class T4R2Processor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); } - private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName) + private void UpdateReport(Guid sourceId) { - const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; - var request = _googleSheetValues.Get(sheetId, "C4:Z4"); - var response = request.Execute(); + try + { + _logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}", + ProcessorType, sourceId); - var r2 = _db.Layers + const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; + + var processedLayer = GetProcessedLayerData(sourceId); + if (processedLayer == null) + { + throw new InvalidOperationException($"Processed layer {sourceId} not found"); + } + + var codesRow = GetCodesFromSheet(sheetId); + + UpdateMonthlyData(sheetId, processedLayer, codesRow); + UpdateYearSummary(sheetId, processedLayer, codesRow); + UpdateTimestamps(sheetId, processedLayer); + UpdateInvoicesData(sheetId, processedLayer); + + _logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports", + ProcessorType); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}", + ProcessorType, sourceId); + throw; + } + } + + private Layer? GetProcessedLayerData(Guid sourceId) + { + return _db.Layers .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) .Include(x => x.Records) .AsNoTracking() .FirstOrDefault(); + } + private IList GetCodesFromSheet(string sheetId) + { + var request = _googleSheetValues.Get(sheetId, "C4:Z4"); + var response = request.Execute(); + return response.Values[0]; + } + + private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList codesRow) + { const int startRow = 6; - var codesRow = response.Values[0]; - for (var i = 1; i <= 12; i++) + for (var month = 1; month <= 12; month++) { var values = new List(); - var month = i < 10 ? $"0{i}" : i.ToString(); - var row = (startRow + i).ToString(); + var monthStr = month < 10 ? $"0{month}" : month.ToString(); + foreach (string code in codesRow) { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}{month}"); - if (record != null) - { - values.Add(record.Value1!.Value); - } - else - { - values.Add("0"); - } + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}"); + values.Add(record?.Value1?.ToString() ?? "0"); } + var valueRange = new ValueRange { Values = new List> { values } }; - var update = _googleSheetValues.Update(valueRange, sheetId, $"{reportSheetName}!C{row}:XZ{row}"); + + var row = (startRow + month).ToString(); + var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}"); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.Execute(); - } - // sum + _logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet", + ProcessorType, month); + } + } + + private void UpdateYearSummary(string sheetId, Layer processedLayer, IList codesRow) + { + const int startRow = 6; var valuesSum = new List(); var emptyRow = new List(); - var rowEmpty = (startRow + 13).ToString(); - var rowSum = (startRow + 14).ToString(); + foreach (string code in codesRow) { - var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); + var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13"); emptyRow.Add(""); - if (record != null) - { - valuesSum.Add(record.Value1!.Value); - } - else - { - valuesSum.Add("0"); - } + valuesSum.Add(record?.Value1?.ToString() ?? "0"); } - // insert empty row before sum + + // Insert empty row before sum + var rowEmpty = (startRow + 13).ToString(); var valueRangeEmpty = new ValueRange { Values = new List> { emptyRow } }; - var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{reportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); + var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateEmpty.Execute(); + // Update sum row + var rowSum = (startRow + 14).ToString(); var valueRangeSum = new ValueRange { Values = new List> { valuesSum } }; - var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{reportSheetName}!C{rowSum}:XZ{rowSum}"); + var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}"); updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateSum.Execute(); - // update time - var timeUtc = new List - { - r2!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; + _logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType); + } + + private void UpdateTimestamps(string sheetId, Layer processedLayer) + { + // Update UTC time + var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeUtcTime = new ValueRange { - Values = new List> { timeUtc } + Values = new List> { new List { timeUtc } } }; - var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{reportSheetName}!G1"); + var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1"); updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeUtc.Execute(); + // Update Warsaw time var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); - var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r2.ModifiedAt.ToUniversalTime(), warsawTimeZone); - var timeWarsaw = new List - { - warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) - }; + var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone); + var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")); var valueRangeWarsawTime = new ValueRange { - Values = new List> { timeWarsaw } + Values = new List> { new List { timeWarsaw } } }; - var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{reportSheetName}!G2"); + var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2"); updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeWarsaw.Execute(); - //invoices + _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}", + ProcessorType, timeUtc, timeWarsaw); + } - var invoices = r2.Records!.Where(x => x.Code!.Length == 12) - .OrderByDescending(x => x.Code); + private void UpdateInvoicesData(string sheetId, Layer processedLayer) + { + var invoices = processedLayer.Records! + .Where(x => x.Code!.Length == 12) + .OrderByDescending(x => x.Code) + .ToList(); var invoicesValues = new List>(); var cleanUpValues = new List>(); + foreach (var invoice in invoices) { - var invoiceDate = - DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) - .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); + var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture) + .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); + var invoiceRow = new List { - invoiceDate, - "", - invoice.Desc1!, - invoice.Value1! + invoiceDate, "", invoice.Desc1!, invoice.Value1! }; invoicesValues.Add(invoiceRow); - var cleanupRow = new List - { - "", "", "", "" - }; + var cleanupRow = new List { "", "", "", "" }; cleanUpValues.Add(cleanupRow); } - + // Clear existing data var cleanupValueRange = new ValueRange { Values = cleanUpValues }; - var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - cleanupInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; cleanupInvoices.Execute(); - + // Update with new data var invoicesValueRange = new ValueRange { Values = invoicesValues }; - var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{invoicesSheetName}!A6:E"); - updateInvoices.ValueInputOption = - SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; + var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E"); + updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateInvoices.Execute(); + _logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet", + ProcessorType, invoices.Count); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs index f58f1ca..73368aa 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs @@ -1,8 +1,8 @@ using Microsoft.EntityFrameworkCore; -using DiunaBI.Core.Models; -using DiunaBI.Core.Database.Context; using Microsoft.Extensions.Logging; using Google.Apis.Sheets.v4; +using DiunaBI.Core.Models; +using DiunaBI.Core.Database.Context; namespace DiunaBI.Plugins.Morska.Processors; @@ -14,6 +14,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + public T4SingleSourceProcessor( AppDbContext db, SpreadsheetsResource.ValuesResource googleSheetValues, @@ -26,70 +32,196 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor public override void Process(Layer processWorker) { - _logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); - - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) + try { - throw new Exception("SourceLayer record not found"); + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); } - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data source + var dataSource = GetDataSource(sourceImportWorker); + + // Process records (simple copy) + var newRecords = ProcessRecords(dataSource); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + if (sourceImportWorker == null) { - throw new Exception("SourceImportWorker layer not found"); + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); } - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (source == null) - { - throw new Exception("Source record not found"); - } + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T4"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; + return processedLayer; + } + private Layer GetDataSource(Layer sourceImportWorker) + { var dataSource = _db.Layers + .Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled) .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .FirstOrDefault(); if (dataSource == null) { - throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); + throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}"); } - var newRecords = dataSource.Records!.Select(record => new Record + _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records", + ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0); + + return dataSource; + } + + private List ProcessRecords(Layer dataSource) + { + if (dataSource.Records == null || dataSource.Records.Count == 0) + { + _logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType); + return new List(); + } + + var newRecords = dataSource.Records.Select(record => new Record { Id = Guid.NewGuid(), Code = record.Code, @@ -99,30 +231,46 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor ModifiedAt = DateTime.UtcNow }).ToList(); - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); } else { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); } SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); - _logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", - processWorker.Name, processWorker.Id); + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -133,6 +281,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 55dcf9b..8bc2d3f 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -13,6 +13,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly ILogger _logger; + // Configuration properties loaded from layer records + private int Year { get; set; } + private int Month { get; set; } + private string? SourceLayer { get; set; } + private string? Source { get; set; } + public T5LastValuesProcessor( AppDbContext db, ILogger logger) @@ -20,97 +26,279 @@ public class T5LastValuesProcessor : MorskaBaseProcessor _db = db; _logger = logger; } + public override void Process(Layer processWorker) { - var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); - var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); - var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; - if (sourceLayer == null) throw new Exception("SourceLayer record not found"); - var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); - if (sourceImportWorker == null) throw new Exception("SourceImportWorker layer not found"); - var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) throw new Exception("Source record not found"); + try + { + _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + // Load configuration from layer records + LoadConfiguration(processWorker); + + // Validate required configuration + ValidateConfiguration(); + + // Perform the actual processing + PerformProcessing(processWorker); + + _logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}", + ProcessorType, processWorker.Name); + } + catch (Exception e) + { + _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})", + ProcessorType, processWorker.Name, processWorker.Id); + throw; + } + } + + private void LoadConfiguration(Layer processWorker) + { + if (processWorker.Records == null) + { + throw new InvalidOperationException("ProcessWorker has no records"); + } + + // Load year + var yearStr = GetRecordValue(processWorker.Records, "Year"); + if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year)) + { + throw new InvalidOperationException("Year record not found or invalid"); + } + Year = year; + + // Load month + var monthStr = GetRecordValue(processWorker.Records, "Month"); + if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month)) + { + throw new InvalidOperationException("Month record not found or invalid"); + } + Month = month; + + // Load source layer + SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer"); + if (string.IsNullOrEmpty(SourceLayer)) + { + throw new InvalidOperationException("SourceLayer record not found"); + } + + // Load source + Source = GetRecordValue(processWorker.Records, "Source"); + if (string.IsNullOrEmpty(Source)) + { + throw new InvalidOperationException("Source record not found"); + } + + _logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}", + ProcessorType, Year, Month, SourceLayer, Source); + } + + private void ValidateConfiguration() + { + var errors = new List(); + + if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}"); + if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}"); + if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required"); + if (string.IsNullOrEmpty(Source)) errors.Add("Source is required"); + + if (errors.Any()) + { + throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}"); + } + + _logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType); + } + + private void PerformProcessing(Layer processWorker) + { + _logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}", + ProcessorType, Year, Month, Source); + + // Get source import worker + var sourceImportWorker = GetSourceImportWorker(); + + // Get or create processed layer + var processedLayer = GetOrCreateProcessedLayer(processWorker); + + // Get data sources + var dataSources = GetDataSources(sourceImportWorker); + + // Process records (get last values for each code) + var newRecords = ProcessRecords(dataSources); + + // Save results + SaveProcessedLayer(processedLayer, newRecords); + + _logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})", + ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id); + } + + private Layer GetSourceImportWorker() + { + var sourceImportWorker = _db.Layers + .Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled) + .FirstOrDefault(); + + if (sourceImportWorker == null) + { + throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}"); + } + + _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})", + ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id); + + return sourceImportWorker; + } + + private Layer GetOrCreateProcessedLayer(Layer processWorker) + { var processedLayer = _db.Layers - .Where(x => x.ParentId == processWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .FirstOrDefault(); - var isNew = false; if (processedLayer == null) { - isNew = true; processedLayer = new Layer { Id = Guid.NewGuid(), Type = LayerType.Processed, ParentId = processWorker.Id, - Number = _db.Layers.Count() + 1 + Number = _db.Layers.Count() + 1, + CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"), + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow }; - processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T5"; - processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5"; + + _logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}", + ProcessorType, processedLayer.Name); + } + else + { processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.CreatedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow; + + _logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}", + ProcessorType, processedLayer.Name); } - processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); - processedLayer.ModifiedAt = DateTime.UtcNow; - - var newRecords = new List(); + return processedLayer; + } + private List GetDataSources(Layer sourceImportWorker) + { var dataSources = _db.Layers .Include(x => x.Records) - .Where(x => x.ParentId == sourceImportWorker.Id - && !x.IsDeleted && !x.IsCancelled) + .Where(x => x.ParentId == sourceImportWorker.Id && + !x.IsDeleted && !x.IsCancelled) .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); - if (dataSources.Count == 0) throw new Exception($"DataSource is empty, {sourceImportWorker.Name}"); + if (dataSources.Count == 0) + { + throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}"); + } - var codes = dataSources.SelectMany(x => x.Records!).Select(x => x.Code).Distinct().ToList(); + _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}", + ProcessorType, dataSources.Count, sourceImportWorker.Name); + + return dataSources; + } + + private List ProcessRecords(List dataSources) + { + var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); + var codes = allRecords.Select(x => x.Code).Distinct().ToList(); + var newRecords = new List(); + + _logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records", + ProcessorType, codes.Count, allRecords.Count); foreach (var code in codes) { - var lastRecord = dataSources.SelectMany(x => x.Records!).Where(x => x.Code == code).OrderByDescending(x => x.CreatedAt).FirstOrDefault(); + var lastRecord = allRecords + .Where(x => x.Code == code) + .OrderByDescending(x => x.CreatedAt) + .FirstOrDefault(); + if (lastRecord == null) continue; - var processedRecord = new Record - { - Id = Guid.NewGuid(), - Code = code, - CreatedAt = DateTime.UtcNow, - ModifiedAt = DateTime.UtcNow - }; - - for (var i = 1; i < 33; i++) - { - if (ProcessHelper.GetValue(lastRecord, i) != null) - { - ProcessHelper.SetValue(processedRecord, i, ProcessHelper.GetValue(lastRecord, i)); - } - } - + var processedRecord = CreateProcessedRecord(lastRecord); newRecords.Add(processedRecord); + + _logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}", + ProcessorType, code, lastRecord.CreatedAt); } - if (isNew) + _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records", + ProcessorType, newRecords.Count); + + return newRecords; + } + + private Record CreateProcessedRecord(Record lastRecord) + { + var processedRecord = new Record + { + Id = Guid.NewGuid(), + Code = lastRecord.Code, + CreatedAt = DateTime.UtcNow, + ModifiedAt = DateTime.UtcNow + }; + + // Copy all values from positions 1-32 + for (var i = 1; i < 33; i++) + { + var value = ProcessHelper.GetValue(lastRecord, i); + if (value != null) + { + ProcessHelper.SetValue(processedRecord, i, value); + } + } + + return processedRecord; + } + + private void SaveProcessedLayer(Layer processedLayer, List newRecords) + { + var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id); + + if (!existsInDb) + { _db.Layers.Add(processedLayer); + _logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType); + } else + { _db.Layers.Update(processedLayer); + _logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType); + } + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}", + ProcessorType, newRecords.Count, processedLayer.Id); } - private void SaveRecords(Guid layerId, ICollection records) + + private void SaveRecords(Guid layerId, ICollection records) { + // Remove existing records for this layer var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); if (toDelete.Count > 0) { _db.Records.RemoveRange(toDelete); + _logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}", + ProcessorType, toDelete.Count, layerId); } + // Add new records foreach (var record in records) { record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); @@ -121,6 +309,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor _db.Records.Add(record); } - _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}", + ProcessorType, records.Count, layerId); + } + + private string? GetRecordValue(ICollection records, string code) + { + return records.FirstOrDefault(x => x.Code == code)?.Desc1; } } \ No newline at end of file