Refactor processors

This commit is contained in:
Michał Zieliński
2025-06-07 13:51:27 +02:00
parent 787df8b838
commit d05dc34e97
14 changed files with 2996 additions and 917 deletions

View File

@@ -26,6 +26,10 @@ public class MorskaD1Importer : MorskaBaseImporter
private DateTime? EndDate { get; set; } private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; } private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaD1Importer( public MorskaD1Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -43,6 +47,10 @@ public class MorskaD1Importer : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
// ✅ Clear cache at start
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker); LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker)) if (!ShouldPerformImport(importWorker))
@@ -65,6 +73,46 @@ public class MorskaD1Importer : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
throw; throw;
} }
finally
{
// ✅ Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
// ✅ Dodaj metodę cache
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
} }
private void LoadConfiguration(Layer importWorker) private void LoadConfiguration(Layer importWorker)
@@ -164,8 +212,8 @@ public class MorskaD1Importer : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); // ✅ Użyj cache zamiast bezpośredniego API
var data = dataRangeResponse.Values; var data = GetSheetData();
if (data == null || data.Count < 2) if (data == null || data.Count < 2)
{ {
@@ -239,10 +287,10 @@ public class MorskaD1Importer : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); // ✅ Użyj cache zamiast bezpośredniego API
var data = dataRangeResponse.Values; var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0); ImporterType, data?.Count ?? 0);
var newRecords = (from t in data var newRecords = (from t in data
@@ -279,8 +327,7 @@ public class MorskaD1Importer : MorskaBaseImporter
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
ImporterType, SheetId);
throw; throw;
} }
} }

View File

@@ -26,6 +26,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
private DateTime? EndDate { get; set; } private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; } private bool IsEnabled { get; set; }
// Cache for sheet data
private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaFk2Importer( public MorskaFk2Importer(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -43,6 +47,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
// ✅ Clear cache at start
_cachedSheetData = null;
_cachedDataKey = null;
LoadConfiguration(importWorker); LoadConfiguration(importWorker);
if (!ShouldPerformImport(importWorker)) if (!ShouldPerformImport(importWorker))
@@ -65,6 +73,46 @@ public class MorskaFk2Importer : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
throw; throw;
} }
finally
{
// ✅ Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
// ✅ Dodaj metodę cache
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
} }
private void LoadConfiguration(Layer importWorker) private void LoadConfiguration(Layer importWorker)
@@ -164,8 +212,8 @@ public class MorskaFk2Importer : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); // ✅ Użyj cache zamiast bezpośredniego API
var data = dataRangeResponse.Values; var data = GetSheetData();
if (data == null || data.Count == 0) if (data == null || data.Count == 0)
{ {
@@ -256,10 +304,10 @@ public class MorskaFk2Importer : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); // ✅ Użyj cache zamiast bezpośredniego API
var data = dataRangeResponse.Values; var data = GetSheetData();
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0); ImporterType, data?.Count ?? 0);
if (data != null) if (data != null)
@@ -315,8 +363,7 @@ public class MorskaFk2Importer : MorskaBaseImporter
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
ImporterType, SheetId);
throw; throw;
} }
} }

View File

@@ -7,13 +7,13 @@ using DiunaBI.Core.Database.Context;
namespace DiunaBI.Plugins.Morska.Importers; namespace DiunaBI.Plugins.Morska.Importers;
public class MorskaImporter : MorskaBaseImporter public class MorskaStandardImporter : MorskaBaseImporter
{ {
public override string ImporterType => "Morska.Import.Standard"; public override string ImporterType => "Morska.Import.Standard";
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<MorskaImporter> _logger; private readonly ILogger<MorskaStandardImporter> _logger;
// Configuration properties // Configuration properties
private string? SheetId { get; set; } private string? SheetId { get; set; }
@@ -26,10 +26,13 @@ public class MorskaImporter : MorskaBaseImporter
private DateTime? EndDate { get; set; } private DateTime? EndDate { get; set; }
private bool IsEnabled { get; set; } private bool IsEnabled { get; set; }
public MorskaImporter( private IList<IList<object>>? _cachedSheetData;
private string? _cachedDataKey;
public MorskaStandardImporter(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<MorskaImporter> logger) ILogger<MorskaStandardImporter> logger)
{ {
_db = db; _db = db;
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
@@ -43,6 +46,10 @@ public class MorskaImporter : MorskaBaseImporter
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})", _logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
// Clear cache before import
_cachedSheetData = null;
_cachedDataKey = null;
// Load configuration from layer records // Load configuration from layer records
LoadConfiguration(importWorker); LoadConfiguration(importWorker);
@@ -69,6 +76,45 @@ public class MorskaImporter : MorskaBaseImporter
ImporterType, importWorker.Name, importWorker.Id); ImporterType, importWorker.Name, importWorker.Id);
throw; throw;
} }
finally
{
// Clear cache after import
_cachedSheetData = null;
_cachedDataKey = null;
}
}
private IList<IList<object>>? GetSheetData()
{
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
{
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
ImporterType, currentDataKey);
return _cachedSheetData;
}
try
{
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
ImporterType, currentDataKey);
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
_cachedSheetData = dataRangeResponse.Values;
_cachedDataKey = currentDataKey;
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
ImporterType, _cachedSheetData?.Count ?? 0);
return _cachedSheetData;
}
catch (Exception e)
{
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
ImporterType, SheetId);
throw;
}
} }
private void LoadConfiguration(Layer importWorker) private void LoadConfiguration(Layer importWorker)
@@ -171,14 +217,13 @@ public class MorskaImporter : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute(); var data = GetSheetData();
var data = dataRangeResponse.Values;
if (data == null || data.Count < 2) if (data == null || data.Count < 2)
{ {
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}", _logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
ImporterType, importWorker.Name); ImporterType, importWorker.Name);
return true; // Assume up to date if no data return true;
} }
// Check if the number of columns matches // Check if the number of columns matches
@@ -199,7 +244,6 @@ public class MorskaImporter : MorskaBaseImporter
continue; continue;
} }
// Check if the record exists in the database - add null check
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString()); var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (existingRecord == null || existingRecord.Value1 != value) if (existingRecord == null || existingRecord.Value1 != value)
{ {
@@ -243,12 +287,9 @@ public class MorskaImporter : MorskaBaseImporter
try try
{ {
var dataRangeResponse = _googleSheetValues.Get( var data = GetSheetData();
SheetId!,
$"{SheetTabName}!{DataRange}").Execute();
var data = dataRangeResponse.Values;
_logger.LogDebug("{ImporterType}: Retrieved {RowCount} rows from Google Sheet", _logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
ImporterType, data?.Count ?? 0); ImporterType, data?.Count ?? 0);
if (data != null && data.Count >= 2) if (data != null && data.Count >= 2)
@@ -284,8 +325,7 @@ public class MorskaImporter : MorskaBaseImporter
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogError(e, "{ImporterType}: Error importing data from Google Sheet {SheetId}", _logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
ImporterType, SheetId);
throw; throw;
} }
} }

View File

@@ -18,6 +18,12 @@ public class T1R1Processor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R1Processor> _logger; private readonly ILogger<T1R1Processor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private List<Record>? Sources { get; set; }
private List<Record>? DynamicCodes { get; set; }
private string? GoogleSheetName { get; set; }
public T1R1Processor( public T1R1Processor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -27,162 +33,321 @@ public class T1R1Processor : MorskaBaseProcessor
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load dynamic codes
DynamicCodes = processWorker.Records
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
// Load Google Sheet name
GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
if (string.IsNullOrEmpty(GoogleSheetName))
{
throw new InvalidOperationException("GoogleSheetName record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}",
ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
ProcessorType, Year, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Process records for each month
var newRecords = ProcessAllMonths();
// Save results
SaveProcessedLayer(processedLayer, newRecords);
// Update Google Sheet report
UpdateGoogleSheetReport(processedLayer.Id);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
private List<Record> ProcessAllMonths()
{
var newRecords = new List<Record>(); var newRecords = new List<Record>();
for (var month = 1; month < 14; month++) for (var month = 1; month < 14; month++)
{ {
if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))) // Skip future months (except month 13 which is summary)
if (Year > DateTime.UtcNow.Year ||
(Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))
{ {
_logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}",
ProcessorType, Year, month);
continue; continue;
} }
_logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}",
ProcessorType, month, Year);
var monthRecords = ProcessSingleMonth(month);
newRecords.AddRange(monthRecords);
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}",
ProcessorType, monthRecords.Count, month);
}
return newRecords;
}
private List<Record> ProcessSingleMonth(int month)
{
var records = new List<Record>(); var records = new List<Record>();
foreach (var source in sources)
{
var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3")
).Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
if (dataSource == null) // Collect records from all sources
foreach (var source in Sources!)
{ {
throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found."); var sourceRecords = GetSourceRecords(source, month);
records.AddRange(sourceRecords);
} }
var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault(); // Process dynamic codes (calculations)
if (codesRecord != null) if (DynamicCodes != null && DynamicCodes.Count > 0)
{ {
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!); var calculatedRecords = ProcessDynamicCodes(records, month);
records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!)))); records.AddRange(calculatedRecords);
}
else
{
records.AddRange(dataSource.Records!);
}
} }
if (dynamicCodes != null) // Create final records with month suffix
{ var monthRecords = records.Select(x => new Record
foreach (var dynamicCode in dynamicCodes)
{
try
{
if (dynamicCode.Desc1 == null)
{
_logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
_logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
try
{
records.Add(calc.CalculateT1(records));
}
catch (Exception e)
{
_logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
_logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
newRecords.AddRange(records.Select(x => new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"{x.Code}{month:D2}", Code = $"{x.Code}{month:D2}",
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32 Value1 = x.Value32
} }).ToList();
));
return monthRecords;
} }
if (isNew) private List<Record> GetSourceRecords(Record source, int month)
{
var dataSource = _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3"))
.Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
if (dataSource == null)
{
throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found");
}
var sourceRecords = new List<Record>();
// Check if there are specific codes configured for this source
var codesRecord = Sources!
.Where(x => x.Code == $"Codes-{source.Desc1}")
.FirstOrDefault();
if (codesRecord != null)
{
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
sourceRecords.AddRange(dataSource.Records!
.Where(x => codes.Contains(int.Parse(x.Code!))));
_logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes",
ProcessorType, source.Desc1, codes.Count);
}
else
{
sourceRecords.AddRange(dataSource.Records!);
_logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records",
ProcessorType, source.Desc1, dataSource.Records!.Count);
}
return sourceRecords;
}
private List<Record> ProcessDynamicCodes(List<Record> records, int month)
{
var calculatedRecords = new List<Record>();
foreach (var dynamicCode in DynamicCodes!)
{
try
{
if (string.IsNullOrEmpty(dynamicCode.Desc1))
{
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}",
ProcessorType, dynamicCode.Id, month);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}",
ProcessorType, calc.Expression, dynamicCode.Id, month);
continue;
}
var calculatedRecord = calc.CalculateT1(records);
calculatedRecords.Add(calculatedRecord);
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}",
ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32);
}
catch (Exception e)
{
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}",
ProcessorType, dynamicCode.Desc1, month);
}
}
return calculatedRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
if (sheetName == null) ProcessorType, newRecords.Count, processedLayer.Id);
{
throw new Exception("GoogleSheetName record not found");
}
UpdateReport(processedLayer.Id, sheetName);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -193,92 +358,139 @@ public class T1R1Processor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
} }
private void UpdateReport(Guid sourceId, string sheetName) private void UpdateGoogleSheetReport(Guid sourceId)
{ {
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; try
var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4"); {
var response = request.Execute(); _logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
var r1 = _db.Layers const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
// Get processed layer data
var processedLayer = _db.Layers
.Where(x => x.Id == sourceId) .Where(x => x.Id == sourceId)
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
var codesRow = response.Values[0]; if (processedLayer == null)
{
throw new InvalidOperationException($"Processed layer {sourceId} not found");
}
// Get codes from sheet header
var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute();
var codesRow = codesResponse.Values[0];
// Update monthly data (months 1-12)
UpdateMonthlyData(sheetId, codesRow, processedLayer);
// Update summary row (month 13)
UpdateSummaryData(sheetId, codesRow, processedLayer);
// Update timestamps
UpdateTimestamps(sheetId, processedLayer);
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
throw;
}
}
private void UpdateMonthlyData(string sheetId, IList<object> codesRow, Layer processedLayer)
{
var valueRange = new ValueRange var valueRange = new ValueRange
{ {
Values = new List<IList<object>>() Values = new List<IList<object>>()
}; };
for (var i = 1; i <= 12; i++) // Process months 1-12
for (var month = 1; month <= 12; month++)
{ {
var values = new List<object>(); var monthValues = new List<object>();
foreach (string code in codesRow) foreach (string code in codesRow)
{ {
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}"); var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}");
if (record != null) monthValues.Add(record?.Value1?.ToString() ?? "0");
{
values.Add(record.Value1!.Value);
} }
else valueRange.Values.Add(monthValues);
{
values.Add("0");
}
}
valueRange.Values.Add(values);
} }
// sum var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18");
var valuesSum = new List<object>(); update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
var emptyRow = new List<object>();
foreach (string code in codesRow)
{
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13");
emptyRow.Add("");
if (record != null)
{
valuesSum.Add(record.Value1!.Value);
}
else
{
valuesSum.Add("0");
}
}
valueRange.Values.Add(emptyRow);
valueRange.Values.Add(valuesSum);
var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20");
update.ValueInputOption =
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute(); update.Execute();
// update time _logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType);
var timeUtc = new List<object> }
private void UpdateSummaryData(string sheetId, IList<object> codesRow, Layer processedLayer)
{ {
r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) var valueRange = new ValueRange
};
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = new List<object>
{
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
};
var valueRangeTime = new ValueRange
{ {
Values = new List<IList<object>>() Values = new List<IList<object>>()
}; };
valueRangeTime.Values.Add(timeUtc);
valueRangeTime.Values.Add(timeWarsaw);
var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); // Empty row
updateTimeUtc.ValueInputOption = var emptyRow = new List<object>();
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; for (int i = 0; i < codesRow.Count; i++)
updateTimeUtc.Execute(); {
emptyRow.Add("");
}
valueRange.Values.Add(emptyRow);
// Summary row (month 13)
var summaryValues = new List<object>();
foreach (string code in codesRow)
{
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
summaryValues.Add(record?.Value1?.ToString() ?? "0");
}
valueRange.Values.Add(summaryValues);
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20");
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute();
_logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType);
}
private void UpdateTimestamps(string sheetId, Layer processedLayer)
{
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
var valueRangeTime = new ValueRange
{
Values = new List<IList<object>>
{
new List<object> { timeUtc },
new List<object> { timeWarsaw }
}
};
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2");
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateTime.Execute();
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
ProcessorType, timeUtc, timeWarsaw);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -18,6 +18,10 @@ public class T1R3Processor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R3Processor> _logger; private readonly ILogger<T1R3Processor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private string? Source { get; set; }
public T1R3Processor( public T1R3Processor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -27,46 +31,140 @@ public class T1R3Processor : MorskaBaseProcessor
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
var source = processWorker.Records?.Where(x => x.Code == "Source").First().Desc1;
if (source == null)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load source
Source = GetRecordValue(processWorker.Records, "Source");
if (string.IsNullOrEmpty(Source))
{
throw new InvalidOperationException("Source record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
ProcessorType, Year, Source);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}",
ProcessorType, Year, Source);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources();
// Process records
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
// Update Google Sheet report
UpdateGoogleSheetReport(processedLayer.Id);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R3-T1"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
private List<Layer> GetDataSources()
{
string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$";
var newRecords = new List<Record>();
string pattern = @$"^L\d+-P-{year}/\d+-{source}-T5$";
var dataSources = _db.Layers var dataSources = _db.Layers
.Where(x => !x.IsDeleted && !x.IsCancelled) .Where(x => !x.IsDeleted && !x.IsCancelled)
.Include(layer => layer.Records!) .Include(layer => layer.Records!)
@@ -75,59 +173,127 @@ public class T1R3Processor : MorskaBaseProcessor
.Where(x => Regex.IsMatch(x.Name!, pattern)) .Where(x => Regex.IsMatch(x.Name!, pattern))
.ToList(); .ToList();
if (dataSources.Count == 0)
{
throw new InvalidOperationException($"No data sources found for pattern: {pattern}");
}
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}",
ProcessorType, dataSources.Count, pattern);
return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var newRecords = new List<Record>();
foreach (var dataSource in dataSources) foreach (var dataSource in dataSources)
{ {
var month = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!); var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!);
if (month == null) if (monthStr == null || !int.TryParse(monthStr, out var month))
{ {
throw new Exception($"Month not found: {dataSource.Name}"); _logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}",
ProcessorType, dataSource.Name);
continue;
} }
_logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}",
ProcessorType, dataSource.Name, month);
var sourceRecords = ProcessDataSourceRecords(dataSource, month);
newRecords.AddRange(sourceRecords);
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}",
ProcessorType, sourceRecords.Count, dataSource.Name);
}
_logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}",
ProcessorType, newRecords.Count);
return newRecords;
}
private List<Record> ProcessDataSourceRecords(Layer dataSource, int month)
{
var newRecords = new List<Record>();
foreach (var record in dataSource.Records!) foreach (var record in dataSource.Records!)
{ {
if (record.Value1 == null) continue; if (record.Value1 == null)
{
_logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null",
ProcessorType, record.Code);
continue;
}
// Process values for positions 1-32
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
if (ProcessHelper.GetValue(record, i) == null) continue; var value = ProcessHelper.GetValue(record, i);
if (value == null)
{
continue;
}
var baseValue = (double)record.Value1!;
var positionValue = (double)value;
var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100;
var newRecord = new Record var newRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"{record.Code}{month}{i:D2}", Code = $"{record.Code}{month:D2}{i:D2}",
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow,
Value1 = i == 1 ? record.Value1 : record.Value1 * ProcessHelper.GetValue(record, i) / 100, Value1 = calculatedValue,
Desc1 = record.Desc1 Desc1 = record.Desc1
}; };
newRecords.Add(newRecord); newRecords.Add(newRecord);
}
_logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}",
ProcessorType, newRecord.Code, newRecord.Value1, record.Code);
} }
} }
if (isNew) return newRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
UpdateReport(processedLayer.Id, year); _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -138,53 +304,118 @@ public class T1R3Processor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
} }
private void UpdateReport(Guid sourceId, int year) private void UpdateGoogleSheetReport(Guid sourceId)
{ {
try
{
_logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}",
ProcessorType, sourceId);
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";
var r3 = _db.Layers var processedLayer = _db.Layers
.Where(x => x.Id == sourceId) .Where(x => x.Id == sourceId)
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
for (var i = 1; i <= 12; i++) if (processedLayer == null)
{ {
var sheetName = ProcessHelper.GetSheetName(i, year); throw new InvalidOperationException($"Processed layer {sourceId} not found");
}
// Update sheets for all months
for (var month = 1; month <= 12; month++)
{
UpdateMonthSheet(sheetId, processedLayer, month);
}
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months",
ProcessorType);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}",
ProcessorType, sourceId);
throw;
}
}
private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month)
{
var sheetName = ProcessHelper.GetSheetName(month, Year);
try
{
_logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}",
ProcessorType, sheetName, month);
// Get codes from sheet
ValueRange? dataRangeResponse; ValueRange? dataRangeResponse;
try try
{ {
dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute(); dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute();
} }
catch catch (Exception e)
{ {
continue; // Sheet not exist _logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}",
ProcessorType, sheetName, e.Message);
return;
} }
if (dataRangeResponse == null) continue; // Sheet not exist if (dataRangeResponse?.Values == null)
var data = dataRangeResponse.Values; {
_logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping",
ProcessorType, sheetName);
return;
}
// Update data
UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month);
// Update timestamps
UpdateSheetTimestamps(sheetId, sheetName, processedLayer);
_logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}",
ProcessorType, sheetName);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}",
ProcessorType, sheetName, month);
throw;
}
}
private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList<IList<object>> codeRows, int month)
{
var updateValueRange = new ValueRange var updateValueRange = new ValueRange
{ {
Values = new List<IList<object>>() Values = new List<IList<object>>()
}; };
foreach (var row in data) foreach (var row in codeRows)
{ {
if (row.Count == 0) continue; if (row.Count == 0) continue;
var code = row[0].ToString();
var code = row[0].ToString();
var updateRow = new List<object>(); var updateRow = new List<object>();
for (var j = 1; j < 16; j++) // Process columns C to Q (positions 1-15)
for (var position = 1; position <= 15; position++)
{ {
var codeRecord = r3!.Records!.FirstOrDefault(x => x.Code == $"{code}{i:D2}{j:D2}"); var recordCode = $"{code}{month:D2}{position:D2}";
if (codeRecord is { Value1: not null }) var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode);
if (codeRecord?.Value1 != null)
{ {
updateRow.Add(codeRecord.Value1); updateRow.Add(codeRecord.Value1);
_logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}",
ProcessorType, codeRecord.Value1, recordCode);
} }
else else
{ {
@@ -195,33 +426,42 @@ public class T1R3Processor : MorskaBaseProcessor
updateValueRange.Values.Add(updateRow); updateValueRange.Values.Add(updateRow);
} }
dataRangeResponse.Values = data; // Update sheet with new values
var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200"); var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:Q200");
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute(); update.Execute();
// update time _logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}",
var timeUtc = new List<object> ProcessorType, updateValueRange.Values.Count, sheetName);
}
private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer)
{ {
r3!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
};
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r3.ModifiedAt.ToUniversalTime(), warsawTimeZone); var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = new List<object> var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
{
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
};
var valueRangeTime = new ValueRange var valueRangeTime = new ValueRange
{ {
Values = new List<IList<object>>() Values = new List<IList<object>>
{
new List<object> { timeUtc },
new List<object> { timeWarsaw }
}
}; };
valueRangeTime.Values.Add(timeUtc);
valueRangeTime.Values.Add(timeWarsaw);
var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2"); var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
updateTimeUtc.ValueInputOption = updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTime.Execute();
updateTimeUtc.Execute();
} _logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
ProcessorType, sheetName, timeUtc, timeWarsaw);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -13,6 +13,13 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesProcessor> _logger; private readonly ILogger<T3MultiSourceCopySelectedCodesProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private int Month { get; set; }
private List<Record>? Sources { get; set; }
private string? Codes { get; set; }
private List<int>? CodesList { get; set; }
public T3MultiSourceCopySelectedCodesProcessor( public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext db, AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesProcessor> logger) ILogger<T3MultiSourceCopySelectedCodesProcessor> logger)
@@ -20,102 +27,268 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
_db = db; _db = db;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
} }
var codes = processWorker.Records?.SingleOrDefault(x => x.Code == "Codes")?.Desc1; catch (Exception e)
if (codes == null)
{ {
throw new Exception("Codes record not found"); _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
var codesList = ProcessHelper.ParseCodes(codes); private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load month
var monthStr = GetRecordValue(processWorker.Records, "Month");
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
{
throw new InvalidOperationException("Month record not found or invalid");
}
Month = month;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load codes
Codes = GetRecordValue(processWorker.Records, "Codes");
if (string.IsNullOrEmpty(Codes))
{
throw new InvalidOperationException("Codes record not found");
}
// Parse codes list
CodesList = ProcessHelper.ParseCodes(Codes);
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}",
ProcessorType, Year, Month, Sources.Count, CodesList.Count);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
ProcessorType, Year, Month, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources();
// Process records
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AB-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
private List<Layer> GetDataSources()
{
var dataSources = new List<Layer>();
foreach (var source in Sources!)
{
var dataSource = _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
.Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
if (dataSource == null)
{
_logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3",
ProcessorType, Year, Month, source.Desc1);
continue;
}
dataSources.Add(dataSource);
_logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
}
var dataSources = sources.Select(source => _db.Layers
.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3"))
.Include(x => x.Records).AsNoTracking()
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception("DataSources are empty"); throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
} }
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources",
ProcessorType, dataSources.Count);
var newRecords = dataSources return dataSources;
.SelectMany(x => x.Records!) }
.Where(x => codesList.Contains(int.Parse(x.Code!)))
.Select(x => private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList();
var filteredRecords = allSourceRecords
.Where(x => !string.IsNullOrEmpty(x.Code) &&
int.TryParse(x.Code, out var code) &&
CodesList!.Contains(code))
.ToList();
_logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes",
ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count);
var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList();
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records",
ProcessorType, newRecords.Count);
return newRecords;
}
private Record CreateCopiedRecord(Record sourceRecord)
{ {
var newRecord = new Record var newRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = x.Code, Code = sourceRecord.Code,
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
// Copy all values from positions 1-32
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.SetValue(newRecord, i, ProcessHelper.GetValue(x, i)); var value = ProcessHelper.GetValue(sourceRecord, i);
ProcessHelper.SetValue(newRecord, i, value);
} }
_logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32",
ProcessorType, newRecord.Code);
return newRecord; return newRecord;
}) }
.ToList();
if (isNew) private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -126,6 +299,12 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -13,6 +13,9 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger; private readonly ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
public T3MultiSourceCopySelectedCodesYearSummaryProcessor( public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
AppDbContext db, AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> logger) ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
@@ -20,67 +23,176 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
_db = db; _db = db;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
{
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
}
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}",
ProcessorType, Year);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}",
ProcessorType, Year);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources for all months
var dataSources = GetDataSources();
// Process records (sum all monthly values)
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AB-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>();
private List<Layer> GetDataSources()
{
var dataSources = new List<Layer>(); var dataSources = new List<Layer>();
for (var i = 1; i < 13; i++) for (var month = 1; month <= 12; month++)
{ {
var j = i; var dataSource = _db.Layers
var dataSource = _db.Layers.Where(x => .Where(x => x.Type == LayerType.Processed &&
x.Type == LayerType.Processed !x.IsDeleted && !x.IsCancelled &&
&& !x.IsDeleted && !x.IsCancelled x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3"))
&& x.Name != null && x.Name.Contains($"{year}/{j:D2}-AB-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
dataSources.Add(dataSource); dataSources.Add(dataSource);
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
}
else
{
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
ProcessorType, month);
} }
} }
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception("DataSources are empty"); throw new InvalidOperationException($"No data sources found for year {Year}");
} }
var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
ProcessorType, dataSources.Count, Year);
foreach (var baseRecord in dataSources.Last().Records!) return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var baseRecords = dataSources.Last().Records!;
var newRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
foreach (var baseRecord in baseRecords)
{ {
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
var processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -88,34 +200,60 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
for (var i = 1; i < 33; i++)
// Sum values from all months for positions 1-32
for (var position = 1; position <= 32; position++)
{ {
ProcessHelper.SetValue(processedRecord, i, var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0);
codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); ProcessHelper.SetValue(processedRecord, position, totalValue);
} }
newRecords.Add(processedRecord); newRecords.Add(processedRecord);
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
ProcessorType, baseRecord.Code, codeRecords.Count);
} }
if (isNew) _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
ProcessorType, newRecords.Count);
return newRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -126,6 +264,12 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -14,6 +14,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceSummaryProcessor> _logger; private readonly ILogger<T3MultiSourceSummaryProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private int Month { get; set; }
private List<Record>? Sources { get; set; }
private List<Record>? DynamicCodes { get; set; }
public T3MultiSourceSummaryProcessor( public T3MultiSourceSummaryProcessor(
AppDbContext db, AppDbContext db,
ILogger<T3MultiSourceSummaryProcessor> logger) ILogger<T3MultiSourceSummaryProcessor> logger)
@@ -24,47 +30,159 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
_logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", try
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load month
var monthStr = GetRecordValue(processWorker.Records, "Month");
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
{
throw new InvalidOperationException("Month record not found or invalid");
}
Month = month;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load dynamic codes
DynamicCodes = processWorker.Records
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
ProcessorType, Year, Month, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources();
// Process records (sum by base codes)
var newRecords = ProcessRecords(dataSources);
// Process dynamic codes if configured
if (DynamicCodes != null && DynamicCodes.Count > 0)
{
var calculatedRecords = ProcessDynamicCodes(newRecords);
newRecords.AddRange(calculatedRecords);
}
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-AA-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>(); private List<Layer> GetDataSources()
{
var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/{month:D2}-{source.Desc1}-T3")) var dataSources = Sources!.Select(source =>
_db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault()) .FirstOrDefault())
@@ -73,17 +191,28 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception("DataSources are empty"); throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
} }
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
ProcessorType, dataSources.Count);
return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
var newRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
foreach (var baseCode in baseCodes) foreach (var baseCode in baseCodes)
{ {
var codeRecords = allRecords.Where(x => var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
x.Code![1..] == baseCode)
.ToList();
var processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -91,82 +220,105 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
// Sum values from all sources for positions 1-32
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.SetValue(processedRecord, i, var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); ProcessHelper.SetValue(processedRecord, i, totalValue);
} }
newRecords.Add(processedRecord); newRecords.Add(processedRecord);
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
ProcessorType, baseCode, codeRecords.Count);
} }
// Dynamic Codes _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
var dynamicCodes = processWorker.Records? ProcessorType, newRecords.Count);
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0) return newRecords;
}
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
{ {
foreach (var dynamicCode in dynamicCodes) var calculatedRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
ProcessorType, DynamicCodes!.Count);
foreach (var dynamicCode in DynamicCodes!)
{ {
try try
{ {
if (dynamicCode.Desc1 == null) if (string.IsNullOrEmpty(dynamicCode.Desc1))
{ {
_logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, dynamicCode.Id);
continue; continue;
} }
var calc = new BaseCalc(dynamicCode.Desc1); var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect()) if (!calc.IsFormulaCorrect())
{ {
_logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, calc.Expression, dynamicCode.Id);
continue; continue;
} }
try var calculatedRecord = calc.CalculateT3(baseRecords);
{ calculatedRecords.Add(calculatedRecord);
newRecords.Add(calc.CalculateT3(newRecords));
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, dynamicCode.Desc1);
}
}
catch (Exception e)
{
_logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
} }
} }
if (isNew) _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
ProcessorType, calculatedRecords.Count);
return calculatedRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
processWorker.Name, processWorker.Id); ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -177,6 +329,12 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -14,6 +14,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger; private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private List<Record>? Sources { get; set; }
private List<Record>? DynamicCodes { get; set; }
public T3MultiSourceYearSummaryProcessor( public T3MultiSourceYearSummaryProcessor(
AppDbContext db, AppDbContext db,
ILogger<T3MultiSourceYearSummaryProcessor> logger) ILogger<T3MultiSourceYearSummaryProcessor> logger)
@@ -24,46 +29,150 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
_logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", try
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load dynamic codes
DynamicCodes = processWorker.Records
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
ProcessorType, Year, Sources.Count, DynamicCodes.Count);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources",
ProcessorType, Year, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources();
// Process records (sum by base codes with validation)
var newRecords = ProcessRecords(dataSources);
// Process dynamic codes if configured
if (DynamicCodes != null && DynamicCodes.Count > 0)
{
var calculatedRecords = ProcessDynamicCodes(newRecords);
newRecords.AddRange(calculatedRecords);
}
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-AA-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>(); private List<Layer> GetDataSources()
{
var dataSources = sources.Select(source => _db.Layers.Where(x => x.Type == LayerType.Processed && !x.IsDeleted && !x.IsCancelled && x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T3")) var dataSources = Sources!.Select(source =>
_db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault()) .FirstOrDefault())
@@ -72,21 +181,28 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception("DataSources are empty"); throw new InvalidOperationException($"No data sources found for year {Year}");
} }
var allRecords = dataSources _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
.SelectMany(x => x.Records!).ToList(); ProcessorType, dataSources.Count, Year);
return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
var newRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
foreach (var baseCode in baseCodes) foreach (var baseCode in baseCodes)
{ {
var codeRecords = allRecords.Where(x => var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
x.Code![1..] == baseCode)
.ToList();
var codeRecordsValidation = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
var processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -94,95 +210,119 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
// Validation record for double-checking calculations
var validationRecord = new Record(); var validationRecord = new Record();
// Sum values from all sources for positions 1-32 with validation
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
ProcessHelper.SetValue(processedRecord, i, var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); ProcessHelper.SetValue(processedRecord, i, totalValue);
ProcessHelper.SetValue(validationRecord, i, // Validation calculation (identical to main calculation)
codeRecordsValidation.Sum(x => ProcessHelper.GetValue(x, i))); var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
ProcessHelper.SetValue(validationRecord, i, validationValue);
if ( // Validate that both calculations match
double.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!);
ProcessHelper.GetValue(validationRecord, i))!) > 0.01) if (difference > 0.01)
{ {
throw new Exception($"ValidationError: Code {baseCode}, " + throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})");
$"Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | " +
$"{ProcessHelper.GetValue(validationRecord, i)})");
} }
} }
newRecords.Add(processedRecord); newRecords.Add(processedRecord);
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
ProcessorType, baseCode, codeRecords.Count);
} }
// Dynamic Codes _logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
var dynamicCodes = processWorker.Records? ProcessorType, newRecords.Count);
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0) return newRecords;
}
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
{ {
foreach (var dynamicCode in dynamicCodes) var calculatedRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
ProcessorType, DynamicCodes!.Count);
foreach (var dynamicCode in DynamicCodes!)
{ {
try try
{ {
if (dynamicCode.Desc1 == null) if (string.IsNullOrEmpty(dynamicCode.Desc1))
{ {
_logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", _logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, dynamicCode.Id);
continue; continue;
} }
var calc = new BaseCalc(dynamicCode.Desc1); var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect()) if (!calc.IsFormulaCorrect())
{ {
_logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", _logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, calc.Expression, dynamicCode.Id);
continue; continue;
} }
try var calculatedRecord = calc.CalculateT3(baseRecords);
{ calculatedRecords.Add(calculatedRecord);
newRecords.Add(calc.CalculateT3(newRecords));
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
} }
catch (Exception e) catch (Exception e)
{ {
_logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", _logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); ProcessorType, dynamicCode.Desc1);
}
}
catch (Exception e)
{
_logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
} }
} }
if (isNew) _logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
ProcessorType, calculatedRecords.Count);
return calculatedRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
processWorker.Name, processWorker.Id); ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -193,6 +333,12 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -15,6 +15,13 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SingleSourceProcessor> _logger; private readonly ILogger<T3SingleSourceProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private int Month { get; set; }
private string? SourceLayerName { get; set; }
private string? Source { get; set; }
private Layer? SourceImportWorker { get; set; }
public T3SingleSourceProcessor( public T3SingleSourceProcessor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -24,70 +31,184 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
_googleSheetValues = googleSheetValues; _googleSheetValues = googleSheetValues;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null)
{ {
throw new Exception("SourceLayer record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
} }
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); catch (Exception e)
if (sourceImportWorker == null)
{ {
throw new Exception("SourceImportWorkerL layer not found"); _logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
} }
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null)
{
throw new Exception("Source record not found");
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year and month
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
var monthStr = GetRecordValue(processWorker.Records, "Month");
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
{
throw new InvalidOperationException("Month record not found or invalid");
}
Month = month;
// Load source layer name
SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer");
if (string.IsNullOrEmpty(SourceLayerName))
{
throw new InvalidOperationException("SourceLayer record not found");
}
// Load source name
Source = GetRecordValue(processWorker.Records, "Source");
if (string.IsNullOrEmpty(Source))
{
throw new InvalidOperationException("Source record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
ProcessorType, Year, Month, SourceLayerName, Source);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required");
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
// Find source import worker
SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled);
if (SourceImportWorker == null)
{
errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found");
}
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
ProcessorType, Year, Month, Source);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources();
// Process records
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>();
private List<Layer> GetDataSources()
{
var dataSources = _db.Layers var dataSources = _db.Layers
.Include(x => x.Records) .Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id .Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderBy(x => x.CreatedAt) .OrderBy(x => x.CreatedAt)
.AsNoTracking() .AsNoTracking()
.ToList(); .ToList();
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception($"DataSources are empty, {sourceImportWorker.Name}"); throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'");
} }
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
ProcessorType, dataSources.Count);
return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var newRecords = new List<Record>();
var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
_logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records",
ProcessorType, allRecords.Count);
foreach (var baseRecord in dataSources.Last().Records!) foreach (var baseRecord in dataSources.Last().Records!)
{ {
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
@@ -100,17 +221,36 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
var lastDayInMonth = DateTime.DaysInMonth(year, month); ProcessDailyValues(processedRecord, codeRecords);
//day 1
newRecords.Add(processedRecord);
}
_logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records",
ProcessorType, newRecords.Count);
return newRecords;
}
private void ProcessDailyValues(Record processedRecord, List<Record> codeRecords)
{
var lastDayInMonth = DateTime.DaysInMonth(Year, Month);
// Day 1 - first value for the month
var firstVal = codeRecords var firstVal = codeRecords
.Where(x => x.CreatedAt.Date <= new DateTime(year, month, 1)).MaxBy(x => x.CreatedAt)?.Value1 ?? 0; .Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1))
.MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
ProcessHelper.SetValue(processedRecord, 1, firstVal); ProcessHelper.SetValue(processedRecord, 1, firstVal);
var previousValue = firstVal; var previousValue = firstVal;
//days 2-29/30
// Days 2 to last-1 - daily differences
for (var i = 2; i < lastDayInMonth; i++) for (var i = 2; i < lastDayInMonth; i++)
{ {
var dayVal = codeRecords var dayVal = codeRecords
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == month).MaxBy(x => x.CreatedAt)?.Value1; .Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month)
.MaxBy(x => x.CreatedAt)?.Value1;
if (dayVal == null) if (dayVal == null)
{ {
ProcessHelper.SetValue(processedRecord, i, 0); ProcessHelper.SetValue(processedRecord, i, 0);
@@ -122,9 +262,11 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
previousValue = (double)dayVal; previousValue = (double)dayVal;
} }
} }
//last day
// Last day - special handling
var lastVal = codeRecords var lastVal = codeRecords
.Where(x => x.CreatedAt.Date >= new DateTime(year, month, lastDayInMonth)).MaxBy(x => x.CreatedAt)?.Value1; .Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth))
.MaxBy(x => x.CreatedAt)?.Value1;
if (lastVal == null) if (lastVal == null)
{ {
@@ -135,32 +277,48 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue); ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
} }
// copy last value // Copy last value to position 32
var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1; var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
ProcessHelper.SetValue(processedRecord, 32, valueToCopy); ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
newRecords.Add(processedRecord); _logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}",
ProcessorType, processedRecord.Code, valueToCopy);
} }
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id);
if (isNew) if (isNew)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -171,7 +329,12 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
} }
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
} }

View File

@@ -15,6 +15,10 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SourceYearSummaryProcessor> _logger; private readonly ILogger<T3SourceYearSummaryProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private string? Source { get; set; }
public T3SourceYearSummaryProcessor( public T3SourceYearSummaryProcessor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -27,72 +31,181 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
_logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", try
processWorker.Name, processWorker.Id);
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load source
Source = GetRecordValue(processWorker.Records, "Source");
if (string.IsNullOrEmpty(Source))
{
throw new InvalidOperationException("Source record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
ProcessorType, Year, Source);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}",
ProcessorType, Year, Source);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources for all months
var dataSources = GetDataSources();
// Process records (sum all monthly values)
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/13-{source}-T3"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>(); private List<Layer> GetDataSources()
var dataSources = new List<Layer>();
for (var i = 1; i < 13; i++)
{ {
var j = i; var dataSources = new List<Layer>();
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed for (var month = 1; month <= 12; month++)
&& !x.IsDeleted && !x.IsCancelled {
&& x.Name != null && x.Name.Contains($"{year}/{j:D2}-{source}-T3")) var dataSource = _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3"))
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) if (dataSource != null)
{ {
dataSources.Add(dataSource); dataSources.Add(dataSource);
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
}
else
{
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
ProcessorType, month);
} }
} }
if (dataSources.Count == 0) if (dataSources.Count == 0)
{ {
throw new Exception("DataSources are empty"); throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}");
} }
var allRecords = dataSources.SelectMany(x => x.Records!).ToList(); _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}",
ProcessorType, dataSources.Count, Year, Source);
foreach (var baseRecord in dataSources.Last().Records!) return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var baseRecords = dataSources.Last().Records!;
var newRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
foreach (var baseRecord in baseRecords)
{ {
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList(); var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
var processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
@@ -100,38 +213,60 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
for (var i = 1; i < 33; i++)
// Sum values from all months for positions 1-32
for (var position = 1; position <= 32; position++)
{ {
ProcessHelper.SetValue(processedRecord, i, var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position));
codeRecords.Sum(x => ProcessHelper.GetValue(x, i))); ProcessHelper.SetValue(processedRecord, position, totalValue);
}
newRecords.Add(processedRecord);
} }
if (isNew) newRecords.Add(processedRecord);
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
ProcessorType, baseRecord.Code, codeRecords.Count);
}
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
ProcessorType, newRecords.Count);
return newRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
processWorker.Name, processWorker.Id); ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -142,6 +277,12 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -7,7 +7,6 @@ using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context; using DiunaBI.Core.Database.Context;
using DiunaBI.Core.Services; using DiunaBI.Core.Services;
namespace DiunaBI.Plugins.Morska.Processors; namespace DiunaBI.Plugins.Morska.Processors;
public class T4R2Processor : MorskaBaseProcessor public class T4R2Processor : MorskaBaseProcessor
@@ -18,6 +17,13 @@ public class T4R2Processor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4R2Processor> _logger; private readonly ILogger<T4R2Processor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private List<Record>? Sources { get; set; }
private string? LayerName { get; set; }
private string? ReportSheetName { get; set; }
private string? InvoicesSheetName { get; set; }
public T4R2Processor( public T4R2Processor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -30,85 +36,274 @@ public class T4R2Processor : MorskaBaseProcessor
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
_logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", try
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
{ {
throw new Exception("Source record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
var layerName = processWorker.Records?.SingleOrDefault(x => x.Code == "LayerName")?.Desc1; private void LoadConfiguration(Layer processWorker)
if (layerName == null)
{ {
throw new Exception("LayerName record not found"); if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
} }
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load layer name
LayerName = GetRecordValue(processWorker.Records, "LayerName");
if (string.IsNullOrEmpty(LayerName))
{
throw new InvalidOperationException("LayerName record not found");
}
// Load report sheet name
ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
if (string.IsNullOrEmpty(ReportSheetName))
{
throw new InvalidOperationException("GoogleSheetName record not found");
}
// Load invoices sheet name
InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices");
if (string.IsNullOrEmpty(InvoicesSheetName))
{
throw new InvalidOperationException("GoogleSheetName-Invoices record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}",
ProcessorType, Year, Sources.Count, LayerName);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required");
if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required");
if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
ProcessorType, Year, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Process records for all sources
var newRecords = ProcessSources();
// Save results
SaveProcessedLayer(processedLayer, newRecords);
// Update Google Sheets reports
UpdateReport(processedLayer.Id);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-{layerName}"; processedLayer.Name = $"L{processedLayer.Number}-{LayerName}";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
private List<Record> ProcessSources()
{
var newRecords = new List<Record>(); var newRecords = new List<Record>();
foreach (var source in sources) foreach (var source in Sources!)
{ {
var rawSourceCodes = processWorker.Records?.SingleOrDefault(x => x.Code == $"Codes-{source.Desc1}") _logger.LogDebug("{ProcessorType}: Processing source {Source}",
?.Desc1; ProcessorType, source.Desc1);
var sourceCodes = new List<int>();
if (rawSourceCodes != null) var sourceCodes = GetSourceCodes(source);
{ var sourceRecords = ProcessSourceData(source, sourceCodes);
sourceCodes = ProcessHelper.ParseCodes(rawSourceCodes); newRecords.AddRange(sourceRecords);
_logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records",
ProcessorType, source.Desc1, sourceRecords.Count);
} }
List<string> lastSourceCodes = []; _logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}",
ProcessorType, newRecords.Count);
return newRecords;
}
private List<int> GetSourceCodes(Record source)
{
var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}");
if (string.IsNullOrEmpty(rawSourceCodes))
{
return new List<int>();
}
return ProcessHelper.ParseCodes(rawSourceCodes);
}
private List<Record> ProcessSourceData(Record source, List<int> sourceCodes)
{
var sourceRecords = new List<Record>();
var lastSourceCodes = new List<string>();
// Process monthly data (1-12)
for (var month = 1; month <= 12; month++) for (var month = 1; month <= 12; month++)
{ {
if ((year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || year < DateTime.UtcNow.Year) var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes);
sourceRecords.AddRange(monthRecords);
}
// Process year summary (month 13)
var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes);
sourceRecords.AddRange(yearSummaryRecords);
return sourceRecords;
}
private List<Record> ProcessMonthData(Record source, List<int> sourceCodes, int month, List<string> lastSourceCodes)
{ {
var monthCopy = month; var monthRecords = new List<Record>();
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed && if (IsDataAvailableForMonth(month))
{
var dataSource = GetMonthDataSource(source, month);
if (dataSource != null)
{
lastSourceCodes.Clear();
lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!));
var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes);
var processedRecords = CreateMonthRecords(filteredRecords, source, month);
monthRecords.AddRange(processedRecords);
_logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records",
ProcessorType, month, source.Desc1, processedRecords.Count);
}
else
{
_logger.LogWarning("{ProcessorType}: Data source {DataSource} not found",
ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T");
}
}
else
{
// Future months - create zero value records (except for FK2)
if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0)
{
var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month);
monthRecords.AddRange(futureRecords);
_logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}",
ProcessorType, futureRecords.Count, month);
}
}
return monthRecords;
}
private bool IsDataAvailableForMonth(int month)
{
return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year;
}
private Layer? GetMonthDataSource(Record source, int month)
{
return _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled && !x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T"))
)
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
if (dataSource != null) }
private List<Record> FilterRecords(IEnumerable<Record> records, List<int> sourceCodes)
{ {
lastSourceCodes = dataSource.Records!.Select(x => x.Code!).ToList(); return records
var news = dataSource.Records!
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) .Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
.Select(x => .ToList();
}
private List<Record> CreateMonthRecords(List<Record> filteredRecords, Record source, int month)
{ {
var newRecord = new Record return filteredRecords.Select(x => new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"{x.Code}{month:D2}", Code = $"{x.Code}{month:D2}",
@@ -116,114 +311,90 @@ public class T4R2Processor : MorskaBaseProcessor
ModifiedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow,
Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1, Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1,
Desc1 = x.Desc1 Desc1 = x.Desc1
}; }).ToList();
return newRecord;
} }
).ToList();
newRecords.AddRange(news); private List<Record> CreateFutureMonthRecords(List<string> lastSourceCodes, List<int> sourceCodes, int month)
}
else
{ {
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", return lastSourceCodes
$"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
}
else
{
//0 values for future months
if (source.Desc1 == "FK2" || lastSourceCodes.Count <= 0) continue;
var news = lastSourceCodes
.Where(x => sourceCodes.Contains(int.Parse(x))) .Where(x => sourceCodes.Contains(int.Parse(x)))
.Select(x => .Select(x => new Record
{
var newRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"{x}{month:D2}", Code = $"{x}{month:D2}",
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow,
Value1 = 0, Value1 = 0
}; }).ToList();
return newRecord;
}
).ToList();
newRecords.AddRange(news);
}
} }
// year summary private List<Record> ProcessYearSummaryData(Record source, List<int> sourceCodes)
var dataSourceSum = _db.Layers.Where(x => {
x.Type == LayerType.Processed && var dataSourceSum = _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled && !x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{year}/13-{source.Desc1}-T") x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T"))
)
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
if (dataSourceSum != null)
if (dataSourceSum == null)
{ {
var news = dataSourceSum.Records! _logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found",
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!))) ProcessorType, $"{Year}/13-{source.Desc1}-T3");
.Select(x => return new List<Record>();
{ }
var newRecord = new Record
var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes);
var yearSummaryRecords = filteredRecords.Select(x => new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = $"{x.Code}13", Code = $"{x.Code}13",
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32 Value1 = x.Value32
}; }).ToList();
return newRecord;
} _logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}",
).ToList(); ProcessorType, yearSummaryRecords.Count, source.Desc1);
newRecords.AddRange(news);
} return yearSummaryRecords;
else
{
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
$"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
} }
if (isNew) private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
if (reportSheetName == null) ProcessorType, newRecords.Count, processedLayer.Id);
{
throw new Exception("GoogleSheetName record not found");
}
var invoicesSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName-Invoices")?.Desc1;
if (invoicesSheetName == null)
{
throw new Exception("GoogleSheetName-Invoices record not found");
}
UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName);
_logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -234,153 +405,196 @@ public class T4R2Processor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
} }
private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName) private void UpdateReport(Guid sourceId)
{ {
const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ"; try
var request = _googleSheetValues.Get(sheetId, "C4:Z4"); {
var response = request.Execute(); _logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}",
ProcessorType, sourceId);
var r2 = _db.Layers const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ";
var processedLayer = GetProcessedLayerData(sourceId);
if (processedLayer == null)
{
throw new InvalidOperationException($"Processed layer {sourceId} not found");
}
var codesRow = GetCodesFromSheet(sheetId);
UpdateMonthlyData(sheetId, processedLayer, codesRow);
UpdateYearSummary(sheetId, processedLayer, codesRow);
UpdateTimestamps(sheetId, processedLayer);
UpdateInvoicesData(sheetId, processedLayer);
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports",
ProcessorType);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}",
ProcessorType, sourceId);
throw;
}
}
private Layer? GetProcessedLayerData(Guid sourceId)
{
return _db.Layers
.Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled) .Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled)
.Include(x => x.Records) .Include(x => x.Records)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
}
private IList<object> GetCodesFromSheet(string sheetId)
{
var request = _googleSheetValues.Get(sheetId, "C4:Z4");
var response = request.Execute();
return response.Values[0];
}
private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList<object> codesRow)
{
const int startRow = 6; const int startRow = 6;
var codesRow = response.Values[0]; for (var month = 1; month <= 12; month++)
for (var i = 1; i <= 12; i++)
{ {
var values = new List<object>(); var values = new List<object>();
var month = i < 10 ? $"0{i}" : i.ToString(); var monthStr = month < 10 ? $"0{month}" : month.ToString();
var row = (startRow + i).ToString();
foreach (string code in codesRow) foreach (string code in codesRow)
{ {
var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}{month}"); var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}");
if (record != null) values.Add(record?.Value1?.ToString() ?? "0");
{
values.Add(record.Value1!.Value);
}
else
{
values.Add("0");
}
} }
var valueRange = new ValueRange var valueRange = new ValueRange
{ {
Values = new List<IList<object>> { values } Values = new List<IList<object>> { values }
}; };
var update = _googleSheetValues.Update(valueRange, sheetId, $"{reportSheetName}!C{row}:XZ{row}");
var row = (startRow + month).ToString();
var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}");
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute(); update.Execute();
_logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet",
ProcessorType, month);
}
} }
// sum private void UpdateYearSummary(string sheetId, Layer processedLayer, IList<object> codesRow)
{
const int startRow = 6;
var valuesSum = new List<object>(); var valuesSum = new List<object>();
var emptyRow = new List<object>(); var emptyRow = new List<object>();
var rowEmpty = (startRow + 13).ToString();
var rowSum = (startRow + 14).ToString();
foreach (string code in codesRow) foreach (string code in codesRow)
{ {
var record = r2!.Records?.SingleOrDefault(x => x.Code == $"{code}13"); var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
emptyRow.Add(""); emptyRow.Add("");
if (record != null) valuesSum.Add(record?.Value1?.ToString() ?? "0");
{
valuesSum.Add(record.Value1!.Value);
} }
else
{ // Insert empty row before sum
valuesSum.Add("0"); var rowEmpty = (startRow + 13).ToString();
}
}
// insert empty row before sum
var valueRangeEmpty = new ValueRange var valueRangeEmpty = new ValueRange
{ {
Values = new List<IList<object>> { emptyRow } Values = new List<IList<object>> { emptyRow }
}; };
var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{reportSheetName}!C{rowEmpty}:XZ{rowEmpty}"); var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}");
updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateEmpty.Execute(); updateEmpty.Execute();
// Update sum row
var rowSum = (startRow + 14).ToString();
var valueRangeSum = new ValueRange var valueRangeSum = new ValueRange
{ {
Values = new List<IList<object>> { valuesSum } Values = new List<IList<object>> { valuesSum }
}; };
var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{reportSheetName}!C{rowSum}:XZ{rowSum}"); var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}");
updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateSum.Execute(); updateSum.Execute();
// update time _logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType);
var timeUtc = new List<object> }
private void UpdateTimestamps(string sheetId, Layer processedLayer)
{ {
r2!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL")) // Update UTC time
}; var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
var valueRangeUtcTime = new ValueRange var valueRangeUtcTime = new ValueRange
{ {
Values = new List<IList<object>> { timeUtc } Values = new List<IList<object>> { new List<object> { timeUtc } }
}; };
var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{reportSheetName}!G1"); var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1");
updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateTimeUtc.Execute(); updateTimeUtc.Execute();
// Update Warsaw time
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time"); var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r2.ModifiedAt.ToUniversalTime(), warsawTimeZone); var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = new List<object> var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
{
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
};
var valueRangeWarsawTime = new ValueRange var valueRangeWarsawTime = new ValueRange
{ {
Values = new List<IList<object>> { timeWarsaw } Values = new List<IList<object>> { new List<object> { timeWarsaw } }
}; };
var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{reportSheetName}!G2"); var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2");
updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED; updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateTimeWarsaw.Execute(); updateTimeWarsaw.Execute();
//invoices _logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
ProcessorType, timeUtc, timeWarsaw);
}
var invoices = r2.Records!.Where(x => x.Code!.Length == 12) private void UpdateInvoicesData(string sheetId, Layer processedLayer)
.OrderByDescending(x => x.Code); {
var invoices = processedLayer.Records!
.Where(x => x.Code!.Length == 12)
.OrderByDescending(x => x.Code)
.ToList();
var invoicesValues = new List<IList<object>>(); var invoicesValues = new List<IList<object>>();
var cleanUpValues = new List<IList<object>>(); var cleanUpValues = new List<IList<object>>();
foreach (var invoice in invoices) foreach (var invoice in invoices)
{ {
var invoiceDate = var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture)
DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture)
.ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL")); .ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL"));
var invoiceRow = new List<object> var invoiceRow = new List<object>
{ {
invoiceDate, invoiceDate, "", invoice.Desc1!, invoice.Value1!
"",
invoice.Desc1!,
invoice.Value1!
}; };
invoicesValues.Add(invoiceRow); invoicesValues.Add(invoiceRow);
var cleanupRow = new List<object> var cleanupRow = new List<object> { "", "", "", "" };
{
"", "", "", ""
};
cleanUpValues.Add(cleanupRow); cleanUpValues.Add(cleanupRow);
} }
// Clear existing data
var cleanupValueRange = new ValueRange { Values = cleanUpValues }; var cleanupValueRange = new ValueRange { Values = cleanUpValues };
var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{invoicesSheetName}!A6:E"); var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
cleanupInvoices.ValueInputOption = cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
cleanupInvoices.Execute(); cleanupInvoices.Execute();
// Update with new data
var invoicesValueRange = new ValueRange { Values = invoicesValues }; var invoicesValueRange = new ValueRange { Values = invoicesValues };
var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{invoicesSheetName}!A6:E"); var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
updateInvoices.ValueInputOption = updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateInvoices.Execute(); updateInvoices.Execute();
_logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet",
ProcessorType, invoices.Count);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -1,8 +1,8 @@
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4;
using DiunaBI.Core.Models;
using DiunaBI.Core.Database.Context;
namespace DiunaBI.Plugins.Morska.Processors; namespace DiunaBI.Plugins.Morska.Processors;
@@ -14,6 +14,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4SingleSourceProcessor> _logger; private readonly ILogger<T4SingleSourceProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private int Month { get; set; }
private string? SourceLayer { get; set; }
private string? Source { get; set; }
public T4SingleSourceProcessor( public T4SingleSourceProcessor(
AppDbContext db, AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues, SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -26,70 +32,196 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
_logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", try
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
if (sourceLayer == null)
{ {
throw new Exception("SourceLayer record not found"); _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
} }
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load month
var monthStr = GetRecordValue(processWorker.Records, "Month");
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
{
throw new InvalidOperationException("Month record not found or invalid");
}
Month = month;
// Load source layer
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
if (string.IsNullOrEmpty(SourceLayer))
{
throw new InvalidOperationException("SourceLayer record not found");
}
// Load source
Source = GetRecordValue(processWorker.Records, "Source");
if (string.IsNullOrEmpty(Source))
{
throw new InvalidOperationException("Source record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
ProcessorType, Year, Month, SourceLayer, Source);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
ProcessorType, Year, Month, Source);
// Get source import worker
var sourceImportWorker = GetSourceImportWorker();
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data source
var dataSource = GetDataSource(sourceImportWorker);
// Process records (simple copy)
var newRecords = ProcessRecords(dataSource);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetSourceImportWorker()
{
var sourceImportWorker = _db.Layers
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
.FirstOrDefault();
if (sourceImportWorker == null) if (sourceImportWorker == null)
{ {
throw new Exception("SourceImportWorker layer not found"); throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
} }
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; _logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
if (source == null) ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
return sourceImportWorker;
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{ {
throw new Exception("Source record not found");
}
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T4"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
private Layer GetDataSource(Layer sourceImportWorker)
{
var dataSource = _db.Layers var dataSource = _db.Layers
.Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled)
.Include(x => x.Records) .Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.AsNoTracking() .AsNoTracking()
.FirstOrDefault(); .FirstOrDefault();
if (dataSource == null) if (dataSource == null)
{ {
throw new Exception($"DataSource not found, {sourceImportWorker.Name}"); throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}");
} }
var newRecords = dataSource.Records!.Select(record => new Record _logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
return dataSource;
}
private List<Record> ProcessRecords(Layer dataSource)
{
if (dataSource.Records == null || dataSource.Records.Count == 0)
{
_logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType);
return new List<Record>();
}
var newRecords = dataSource.Records.Select(record => new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = record.Code, Code = record.Code,
@@ -99,30 +231,46 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}).ToList(); }).ToList();
if (isNew) _logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source",
ProcessorType, newRecords.Count);
return newRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{ {
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
} }
else else
{ {
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
} }
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", _logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
processWorker.Name, processWorker.Id); ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -133,6 +281,12 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }

View File

@@ -13,6 +13,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
private readonly AppDbContext _db; private readonly AppDbContext _db;
private readonly ILogger<T5LastValuesProcessor> _logger; private readonly ILogger<T5LastValuesProcessor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private int Month { get; set; }
private string? SourceLayer { get; set; }
private string? Source { get; set; }
public T5LastValuesProcessor( public T5LastValuesProcessor(
AppDbContext db, AppDbContext db,
ILogger<T5LastValuesProcessor> logger) ILogger<T5LastValuesProcessor> logger)
@@ -20,97 +26,279 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
_db = db; _db = db;
_logger = logger; _logger = logger;
} }
public override void Process(Layer processWorker) public override void Process(Layer processWorker)
{ {
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); try
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); {
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; _logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
if (sourceLayer == null) throw new Exception("SourceLayer record not found"); ProcessorType, processWorker.Name, processWorker.Id);
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
if (sourceImportWorker == null) throw new Exception("SourceImportWorker layer not found");
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null) throw new Exception("Source record not found");
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
}
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load month
var monthStr = GetRecordValue(processWorker.Records, "Month");
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
{
throw new InvalidOperationException("Month record not found or invalid");
}
Month = month;
// Load source layer
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
if (string.IsNullOrEmpty(SourceLayer))
{
throw new InvalidOperationException("SourceLayer record not found");
}
// Load source
Source = GetRecordValue(processWorker.Records, "Source");
if (string.IsNullOrEmpty(Source))
{
throw new InvalidOperationException("Source record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
ProcessorType, Year, Month, SourceLayer, Source);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
ProcessorType, Year, Month, Source);
// Get source import worker
var sourceImportWorker = GetSourceImportWorker();
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Get data sources
var dataSources = GetDataSources(sourceImportWorker);
// Process records (get last values for each code)
var newRecords = ProcessRecords(dataSources);
// Save results
SaveProcessedLayer(processedLayer, newRecords);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetSourceImportWorker()
{
var sourceImportWorker = _db.Layers
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
.FirstOrDefault();
if (sourceImportWorker == null)
{
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
}
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
return sourceImportWorker;
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
&& !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.FirstOrDefault(); .FirstOrDefault();
var isNew = false;
if (processedLayer == null) if (processedLayer == null)
{ {
isNew = true;
processedLayer = new Layer processedLayer = new Layer
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Type = LayerType.Processed, Type = LayerType.Processed,
ParentId = processWorker.Id, ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1 Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
}; };
processedLayer.Name = $"L{processedLayer.Number}-P-{year}/{month:D2}-{source}-T5"; processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow; processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
} }
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); return processedLayer;
processedLayer.ModifiedAt = DateTime.UtcNow; }
var newRecords = new List<Record>();
private List<Layer> GetDataSources(Layer sourceImportWorker)
{
var dataSources = _db.Layers var dataSources = _db.Layers
.Include(x => x.Records) .Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id .Where(x => x.ParentId == sourceImportWorker.Id &&
&& !x.IsDeleted && !x.IsCancelled) !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt) .OrderByDescending(x => x.CreatedAt)
.AsNoTracking() .AsNoTracking()
.ToList(); .ToList();
if (dataSources.Count == 0) throw new Exception($"DataSource is empty, {sourceImportWorker.Name}"); if (dataSources.Count == 0)
{
throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}");
}
var codes = dataSources.SelectMany(x => x.Records!).Select(x => x.Code).Distinct().ToList(); _logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}",
ProcessorType, dataSources.Count, sourceImportWorker.Name);
return dataSources;
}
private List<Record> ProcessRecords(List<Layer> dataSources)
{
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
var codes = allRecords.Select(x => x.Code).Distinct().ToList();
var newRecords = new List<Record>();
_logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records",
ProcessorType, codes.Count, allRecords.Count);
foreach (var code in codes) foreach (var code in codes)
{ {
var lastRecord = dataSources.SelectMany(x => x.Records!).Where(x => x.Code == code).OrderByDescending(x => x.CreatedAt).FirstOrDefault(); var lastRecord = allRecords
.Where(x => x.Code == code)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
if (lastRecord == null) continue; if (lastRecord == null) continue;
var processedRecord = CreateProcessedRecord(lastRecord);
newRecords.Add(processedRecord);
_logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}",
ProcessorType, code, lastRecord.CreatedAt);
}
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records",
ProcessorType, newRecords.Count);
return newRecords;
}
private Record CreateProcessedRecord(Record lastRecord)
{
var processedRecord = new Record var processedRecord = new Record
{ {
Id = Guid.NewGuid(), Id = Guid.NewGuid(),
Code = code, Code = lastRecord.Code,
CreatedAt = DateTime.UtcNow, CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow ModifiedAt = DateTime.UtcNow
}; };
// Copy all values from positions 1-32
for (var i = 1; i < 33; i++) for (var i = 1; i < 33; i++)
{ {
if (ProcessHelper.GetValue(lastRecord, i) != null) var value = ProcessHelper.GetValue(lastRecord, i);
if (value != null)
{ {
ProcessHelper.SetValue(processedRecord, i, ProcessHelper.GetValue(lastRecord, i)); ProcessHelper.SetValue(processedRecord, i, value);
} }
} }
newRecords.Add(processedRecord); return processedRecord;
} }
if (isNew) private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{
_db.Layers.Add(processedLayer); _db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
}
else else
{
_db.Layers.Update(processedLayer); _db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
}
SaveRecords(processedLayer.Id, newRecords); SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges(); _db.SaveChanges();
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
} }
private void SaveRecords(Guid layerId, ICollection<Record> records) private void SaveRecords(Guid layerId, ICollection<Record> records)
{ {
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0) if (toDelete.Count > 0)
{ {
_db.Records.RemoveRange(toDelete); _db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
} }
// Add new records
foreach (var record in records) foreach (var record in records)
{ {
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -121,6 +309,12 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
_db.Records.Add(record); _db.Records.Add(record);
} }
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); _logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
} }
} }