mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor processors

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-07 13:51:27 +02:00
parent 787df8b838
commit d05dc34e97
14 changed files with 2996 additions and 917 deletions
Download Patch File Download Diff File Expand all files Collapse all files

544
src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs
View File

@@ -18,6 +18,12 @@ public class T1R1Processor : MorskaBaseProcessor
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R1Processor> _logger;
// Configuration properties loaded from layer records
private int Year { get; set; }
private List<Record>? Sources { get; set; }
private List<Record>? DynamicCodes { get; set; }
private string? GoogleSheetName { get; set; }
public T1R1Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
@@ -27,162 +33,321 @@ public class T1R1Processor : MorskaBaseProcessor
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
try
{
throw new Exception("Source record not found");
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
// Load configuration from layer records
LoadConfiguration(processWorker);
// Validate required configuration
ValidateConfiguration();
// Perform the actual processing
PerformProcessing(processWorker);
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
ProcessorType, processWorker.Name);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
ProcessorType, processWorker.Name, processWorker.Id);
throw;
}
}
private void LoadConfiguration(Layer processWorker)
{
if (processWorker.Records == null)
{
throw new InvalidOperationException("ProcessWorker has no records");
}
// Load year
var yearStr = GetRecordValue(processWorker.Records, "Year");
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
{
throw new InvalidOperationException("Year record not found or invalid");
}
Year = year;
// Load sources
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
if (Sources.Count == 0)
{
throw new InvalidOperationException("Source records not found");
}
// Load dynamic codes
DynamicCodes = processWorker.Records
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
// Load Google Sheet name
GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
if (string.IsNullOrEmpty(GoogleSheetName))
{
throw new InvalidOperationException("GoogleSheetName record not found");
}
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}",
ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName);
}
private void ValidateConfiguration()
{
var errors = new List<string>();
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required");
if (errors.Any())
{
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
}
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
}
private void PerformProcessing(Layer processWorker)
{
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
ProcessorType, Year, Sources!.Count);
// Get or create processed layer
var processedLayer = GetOrCreateProcessedLayer(processWorker);
// Process records for each month
var newRecords = ProcessAllMonths();
// Save results
SaveProcessedLayer(processedLayer, newRecords);
// Update Google Sheet report
UpdateGoogleSheetReport(processedLayer.Id);
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
}
private Layer GetOrCreateProcessedLayer(Layer processWorker)
{
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
.OrderByDescending(x => x.CreatedAt)
.FirstOrDefault();
var isNew = false;
if (processedLayer == null)
{
isNew = true;
processedLayer = new Layer
{
Id = Guid.NewGuid(),
Type = LayerType.Processed,
ParentId = processWorker.Id,
Number = _db.Layers.Count() + 1
Number = _db.Layers.Count() + 1,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
};
processedLayer.Name = $"L{processedLayer.Number}-P-{year}-R1-T1";
processedLayer.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1";
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
else
{
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.CreatedAt = DateTime.UtcNow;
processedLayer.ModifiedAt = DateTime.UtcNow;
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
ProcessorType, processedLayer.Name);
}
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
var dynamicCodes = processWorker.Records?.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
.ToList();
return processedLayer;
}
private List<Record> ProcessAllMonths()
{
var newRecords = new List<Record>();
for (var month = 1; month < 14; month++)
{
if (year > DateTime.UtcNow.Year || ((year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13)))
// Skip future months (except month 13 which is summary)
if (Year > DateTime.UtcNow.Year ||
(Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))
{
_logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}",
ProcessorType, Year, month);
continue;
}
var records = new List<Record>();
foreach (var source in sources)
{
var monthCopy = month;
var dataSource = _db.Layers.Where(x =>
x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{year}/{monthCopy:D2}-{source.Desc1}-T3")
).Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
_logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}",
ProcessorType, month, Year);
if (dataSource == null)
{
throw new Exception($"Source layer {year}/{monthCopy}-{source.Desc1}-T3 not found.");
}
var monthRecords = ProcessSingleMonth(month);
newRecords.AddRange(monthRecords);
var codesRecord = processWorker.Records?.Where(x => x.Code == $"Codes-{source.Desc1}").FirstOrDefault();
if (codesRecord != null)
{
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
records.AddRange(dataSource.Records!.Where(x => codes.Contains(int.Parse(x.Code!))));
}
else
{
records.AddRange(dataSource.Records!);
}
}
if (dynamicCodes != null)
{
foreach (var dynamicCode in dynamicCodes)
{
try
{
if (dynamicCode.Desc1 == null)
{
_logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
_logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
try
{
records.Add(calc.CalculateT1(records));
}
catch (Exception e)
{
_logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
_logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
newRecords.AddRange(records.Select(x => new Record
{
Id = Guid.NewGuid(),
Code = $"{x.Code}{month:D2}",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
}
));
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}",
ProcessorType, monthRecords.Count, month);
}
if (isNew)
return newRecords;
}
private List<Record> ProcessSingleMonth(int month)
{
var records = new List<Record>();
// Collect records from all sources
foreach (var source in Sources!)
{
var sourceRecords = GetSourceRecords(source, month);
records.AddRange(sourceRecords);
}
// Process dynamic codes (calculations)
if (DynamicCodes != null && DynamicCodes.Count > 0)
{
var calculatedRecords = ProcessDynamicCodes(records, month);
records.AddRange(calculatedRecords);
}
// Create final records with month suffix
var monthRecords = records.Select(x => new Record
{
Id = Guid.NewGuid(),
Code = $"{x.Code}{month:D2}",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
Value1 = x.Value32
}).ToList();
return monthRecords;
}
private List<Record> GetSourceRecords(Record source, int month)
{
var dataSource = _db.Layers
.Where(x => x.Type == LayerType.Processed &&
!x.IsDeleted && !x.IsCancelled &&
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3"))
.Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
if (dataSource == null)
{
throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found");
}
var sourceRecords = new List<Record>();
// Check if there are specific codes configured for this source
var codesRecord = Sources!
.Where(x => x.Code == $"Codes-{source.Desc1}")
.FirstOrDefault();
if (codesRecord != null)
{
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
sourceRecords.AddRange(dataSource.Records!
.Where(x => codes.Contains(int.Parse(x.Code!))));
_logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes",
ProcessorType, source.Desc1, codes.Count);
}
else
{
sourceRecords.AddRange(dataSource.Records!);
_logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records",
ProcessorType, source.Desc1, dataSource.Records!.Count);
}
return sourceRecords;
}
private List<Record> ProcessDynamicCodes(List<Record> records, int month)
{
var calculatedRecords = new List<Record>();
foreach (var dynamicCode in DynamicCodes!)
{
try
{
if (string.IsNullOrEmpty(dynamicCode.Desc1))
{
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}",
ProcessorType, dynamicCode.Id, month);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}",
ProcessorType, calc.Expression, dynamicCode.Id, month);
continue;
}
var calculatedRecord = calc.CalculateT1(records);
calculatedRecords.Add(calculatedRecord);
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}",
ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32);
}
catch (Exception e)
{
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}",
ProcessorType, dynamicCode.Desc1, month);
}
}
return calculatedRecords;
}
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
{
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
if (!existsInDb)
{
_db.Layers.Add(processedLayer);
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
}
else
{
_db.Layers.Update(processedLayer);
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
}
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
if (sheetName == null)
{
throw new Exception("GoogleSheetName record not found");
}
UpdateReport(processedLayer.Id, sheetName);
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
ProcessorType, newRecords.Count, processedLayer.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
// Remove existing records for this layer
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
ProcessorType, toDelete.Count, layerId);
}
// Add new records
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
@@ -193,92 +358,139 @@ public class T1R1Processor : MorskaBaseProcessor
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
ProcessorType, records.Count, layerId);
}
private void UpdateReport(Guid sourceId, string sheetName)
private void UpdateGoogleSheetReport(Guid sourceId)
{
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
var request = _googleSheetValues.Get(sheetId, $"{sheetName}!C4:DC4");
var response = request.Execute();
try
{
_logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
var r1 = _db.Layers
.Where(x => x.Id == sourceId)
.Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
// Get processed layer data
var processedLayer = _db.Layers
.Where(x => x.Id == sourceId)
.Include(x => x.Records)
.AsNoTracking()
.FirstOrDefault();
var codesRow = response.Values[0];
if (processedLayer == null)
{
throw new InvalidOperationException($"Processed layer {sourceId} not found");
}
// Get codes from sheet header
var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DC4").Execute();
var codesRow = codesResponse.Values[0];
// Update monthly data (months 1-12)
UpdateMonthlyData(sheetId, codesRow, processedLayer);
// Update summary row (month 13)
UpdateSummaryData(sheetId, codesRow, processedLayer);
// Update timestamps
UpdateTimestamps(sheetId, processedLayer);
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
}
catch (Exception e)
{
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}",
ProcessorType, GoogleSheetName);
throw;
}
}
private void UpdateMonthlyData(string sheetId, IList<object> codesRow, Layer processedLayer)
{
var valueRange = new ValueRange
{
Values = new List<IList<object>>()
};
for (var i = 1; i <= 12; i++)
// Process months 1-12
for (var month = 1; month <= 12; month++)
{
var values = new List<object>();
var monthValues = new List<object>();
foreach (string code in codesRow)
{
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}{i:D2}");
if (record != null)
{
values.Add(record.Value1!.Value);
}
else
{
values.Add("0");
}
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}");
monthValues.Add(record?.Value1?.ToString() ?? "0");
}
valueRange.Values.Add(values);
valueRange.Values.Add(monthValues);
}
// sum
var valuesSum = new List<object>();
var emptyRow = new List<object>();
foreach (string code in codesRow)
{
var record = r1!.Records?.SingleOrDefault(x => x.Code == $"{code}13");
emptyRow.Add("");
if (record != null)
{
valuesSum.Add(record.Value1!.Value);
}
else
{
valuesSum.Add("0");
}
}
valueRange.Values.Add(emptyRow);
valueRange.Values.Add(valuesSum);
var update = _googleSheetValues.Update(valueRange, sheetId, $"{sheetName}!C7:DC20");
update.ValueInputOption =
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DC18");
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute();
// update time
var timeUtc = new List<object>
{
r1!.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
};
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(r1.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = new List<object>
{
warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"))
};
var valueRangeTime = new ValueRange
_logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType);
}
private void UpdateSummaryData(string sheetId, IList<object> codesRow, Layer processedLayer)
{
var valueRange = new ValueRange
{
Values = new List<IList<object>>()
};
valueRangeTime.Values.Add(timeUtc);
valueRangeTime.Values.Add(timeWarsaw);
var updateTimeUtc = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
updateTimeUtc.ValueInputOption =
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateTimeUtc.Execute();
// Empty row
var emptyRow = new List<object>();
for (int i = 0; i < codesRow.Count; i++)
{
emptyRow.Add("");
}
valueRange.Values.Add(emptyRow);
// Summary row (month 13)
var summaryValues = new List<object>();
foreach (string code in codesRow)
{
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
summaryValues.Add(record?.Value1?.ToString() ?? "0");
}
valueRange.Values.Add(summaryValues);
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DC20");
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
update.Execute();
_logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType);
}
private void UpdateTimestamps(string sheetId, Layer processedLayer)
{
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
var valueRangeTime = new ValueRange
{
Values = new List<IList<object>>
{
new List<object> { timeUtc },
new List<object> { timeWarsaw }
}
};
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2");
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
updateTime.Execute();
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
ProcessorType, timeUtc, timeWarsaw);
}
private string? GetRecordValue(ICollection<Record> records, string code)
{
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
}
}