344 lines
13 KiB
C#
344 lines
13 KiB
C#
using DiunaBI.Domain.Entities;
|
|
using DiunaBI.Infrastructure.Data;
|
|
using DiunaBI.Infrastructure.Services;
|
|
using DiunaBI.Infrastructure.Services.Calculations;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace DiunaBI.Plugins.Morska.Processors;
|
|
|
|
public class MorskaT3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
|
|
{
|
|
public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary";
|
|
|
|
private readonly AppDbContext _db;
|
|
private readonly ILogger<MorskaT3MultiSourceYearSummaryProcessor> _logger;
|
|
|
|
// Configuration properties loaded from layer records
|
|
private int Year { get; set; }
|
|
private List<Record>? Sources { get; set; }
|
|
private List<Record>? DynamicCodes { get; set; }
|
|
|
|
public MorskaT3MultiSourceYearSummaryProcessor(
|
|
AppDbContext db,
|
|
ILogger<MorskaT3MultiSourceYearSummaryProcessor> logger)
|
|
{
|
|
_db = db;
|
|
_logger = logger;
|
|
}
|
|
|
|
public override void Process(Layer processWorker)
|
|
{
|
|
try
|
|
{
|
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|
ProcessorType, processWorker.Name, processWorker.Id);
|
|
|
|
// Load configuration from layer records
|
|
LoadConfiguration(processWorker);
|
|
|
|
// Validate required configuration
|
|
ValidateConfiguration();
|
|
|
|
// Perform the actual processing
|
|
PerformProcessing(processWorker);
|
|
|
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
|
ProcessorType, processWorker.Name);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
|
ProcessorType, processWorker.Name, processWorker.Id);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private void LoadConfiguration(Layer processWorker)
|
|
{
|
|
if (processWorker.Records == null)
|
|
{
|
|
throw new InvalidOperationException("ProcessWorker has no records");
|
|
}
|
|
|
|
// Load year
|
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
|
{
|
|
throw new InvalidOperationException("Year record not found or invalid");
|
|
}
|
|
Year = year;
|
|
|
|
// Load sources
|
|
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
|
if (Sources.Count == 0)
|
|
{
|
|
throw new InvalidOperationException("Source records not found");
|
|
}
|
|
|
|
// Load dynamic codes
|
|
DynamicCodes = processWorker.Records
|
|
.Where(x => x.Code!.Contains("DynamicCode-"))
|
|
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
|
.ToList();
|
|
|
|
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
|
|
ProcessorType, Year, Sources.Count, DynamicCodes.Count);
|
|
}
|
|
|
|
private void ValidateConfiguration()
|
|
{
|
|
var errors = new List<string>();
|
|
|
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
|
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
|
|
|
if (errors.Any())
|
|
{
|
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
|
}
|
|
|
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
|
}
|
|
|
|
private void PerformProcessing(Layer processWorker)
|
|
{
|
|
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources",
|
|
ProcessorType, Year, Sources!.Count);
|
|
|
|
// Get or create processed layer
|
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
|
|
|
// Get data sources
|
|
var dataSources = GetDataSources();
|
|
|
|
// Process records (sum by base codes with validation)
|
|
var newRecords = ProcessRecords(dataSources);
|
|
|
|
// Process dynamic codes if configured
|
|
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
|
{
|
|
var calculatedRecords = ProcessDynamicCodes(newRecords);
|
|
newRecords.AddRange(calculatedRecords);
|
|
}
|
|
|
|
// Save results
|
|
SaveProcessedLayer(processedLayer, newRecords);
|
|
|
|
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
|
}
|
|
|
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
|
{
|
|
var processedLayer = _db.Layers
|
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
|
.OrderByDescending(x => x.CreatedAt)
|
|
.FirstOrDefault();
|
|
|
|
if (processedLayer == null)
|
|
{
|
|
processedLayer = new Layer
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Type = LayerType.Processed,
|
|
ParentId = processWorker.Id,
|
|
Number = _db.Layers.Count() + 1,
|
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|
CreatedAt = DateTime.UtcNow,
|
|
ModifiedAt = DateTime.UtcNow
|
|
};
|
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3";
|
|
|
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
|
ProcessorType, processedLayer.Name);
|
|
}
|
|
else
|
|
{
|
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|
|
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
|
ProcessorType, processedLayer.Name);
|
|
}
|
|
|
|
return processedLayer;
|
|
}
|
|
|
|
private List<Layer> GetDataSources()
|
|
{
|
|
var dataSources = Sources!.Select(source =>
|
|
_db.Layers
|
|
.Where(x => x.Type == LayerType.Processed &&
|
|
!x.IsDeleted && !x.IsCancelled &&
|
|
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3"))
|
|
.Include(x => x.Records)
|
|
.AsNoTracking()
|
|
.FirstOrDefault())
|
|
.OfType<Layer>()
|
|
.ToList();
|
|
|
|
if (dataSources.Count == 0)
|
|
{
|
|
throw new InvalidOperationException($"No data sources found for year {Year}");
|
|
}
|
|
|
|
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
|
|
ProcessorType, dataSources.Count, Year);
|
|
|
|
return dataSources;
|
|
}
|
|
|
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
|
{
|
|
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
|
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
|
var newRecords = new List<Record>();
|
|
|
|
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
|
|
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
|
|
|
|
foreach (var baseCode in baseCodes)
|
|
{
|
|
var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
|
|
|
|
var processedRecord = new Record
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Code = $"9{baseCode}",
|
|
CreatedAt = DateTime.UtcNow,
|
|
ModifiedAt = DateTime.UtcNow
|
|
};
|
|
|
|
// Validation record for double-checking calculations
|
|
var validationRecord = new Record();
|
|
|
|
// Sum values from all sources for positions 1-32 with validation
|
|
for (var i = 1; i < 33; i++)
|
|
{
|
|
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
|
ProcessHelper.SetValue(processedRecord, i, totalValue);
|
|
|
|
// Validation calculation (identical to main calculation)
|
|
var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
|
ProcessHelper.SetValue(validationRecord, i, validationValue);
|
|
|
|
// Validate that both calculations match
|
|
var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!);
|
|
if (difference > 0.01)
|
|
{
|
|
throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})");
|
|
}
|
|
}
|
|
|
|
newRecords.Add(processedRecord);
|
|
|
|
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
|
|
ProcessorType, baseCode, codeRecords.Count);
|
|
}
|
|
|
|
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
|
ProcessorType, newRecords.Count);
|
|
|
|
return newRecords;
|
|
}
|
|
|
|
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
|
|
{
|
|
var calculatedRecords = new List<Record>();
|
|
|
|
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
|
|
ProcessorType, DynamicCodes!.Count);
|
|
|
|
foreach (var dynamicCode in DynamicCodes!)
|
|
{
|
|
try
|
|
{
|
|
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
|
{
|
|
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
|
|
ProcessorType, dynamicCode.Id);
|
|
continue;
|
|
}
|
|
|
|
var calc = new BaseCalc(dynamicCode.Desc1);
|
|
if (!calc.IsFormulaCorrect())
|
|
{
|
|
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
|
|
ProcessorType, calc.Expression, dynamicCode.Id);
|
|
continue;
|
|
}
|
|
|
|
var calculatedRecord = calc.CalculateT3(baseRecords.Concat(calculatedRecords).ToList());
|
|
calculatedRecords.Add(calculatedRecord);
|
|
|
|
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
|
|
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
|
|
ProcessorType, dynamicCode.Desc1);
|
|
}
|
|
}
|
|
|
|
_logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
|
|
ProcessorType, calculatedRecords.Count);
|
|
|
|
return calculatedRecords;
|
|
}
|
|
|
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
|
{
|
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
|
|
|
if (!existsInDb)
|
|
{
|
|
_db.Layers.Add(processedLayer);
|
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
|
}
|
|
else
|
|
{
|
|
_db.Layers.Update(processedLayer);
|
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
|
}
|
|
|
|
SaveRecords(processedLayer.Id, newRecords);
|
|
_db.SaveChanges();
|
|
|
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
|
}
|
|
|
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|
{
|
|
// Remove existing records for this layer
|
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|
if (toDelete.Count > 0)
|
|
{
|
|
_db.Records.RemoveRange(toDelete);
|
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
|
ProcessorType, toDelete.Count, layerId);
|
|
}
|
|
|
|
// Add new records
|
|
foreach (var record in records)
|
|
{
|
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
record.CreatedAt = DateTime.UtcNow;
|
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|
record.ModifiedAt = DateTime.UtcNow;
|
|
record.LayerId = layerId;
|
|
_db.Records.Add(record);
|
|
}
|
|
|
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
|
ProcessorType, records.Count, layerId);
|
|
}
|
|
|
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
|
{
|
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
|
}
|
|
} |