after refactor cleanup
This commit is contained in:
322
DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs
Normal file
322
DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs
Normal file
@@ -0,0 +1,322 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT5LastValuesProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T5.LastValues";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT5LastValuesProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private string? SourceLayer { get; set; }
|
||||
private string? Source { get; set; }
|
||||
|
||||
public MorskaT5LastValuesProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT5LastValuesProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load month
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load source layer
|
||||
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||
if (string.IsNullOrEmpty(SourceLayer))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayer record not found");
|
||||
}
|
||||
|
||||
// Load source
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||
ProcessorType, Year, Month, SourceLayer, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||
ProcessorType, Year, Month, Source);
|
||||
|
||||
// Get source import worker
|
||||
var sourceImportWorker = GetSourceImportWorker();
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources(sourceImportWorker);
|
||||
|
||||
// Process records (get last values for each code)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetSourceImportWorker()
|
||||
{
|
||||
var sourceImportWorker = _db.Layers
|
||||
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (sourceImportWorker == null)
|
||||
{
|
||||
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
|
||||
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
|
||||
|
||||
return sourceImportWorker;
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources(Layer sourceImportWorker)
|
||||
{
|
||||
var dataSources = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == sourceImportWorker.Id &&
|
||||
!x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}",
|
||||
ProcessorType, dataSources.Count, sourceImportWorker.Name);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var codes = allRecords.Select(x => x.Code).Distinct().ToList();
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records",
|
||||
ProcessorType, codes.Count, allRecords.Count);
|
||||
|
||||
foreach (var code in codes)
|
||||
{
|
||||
var lastRecord = allRecords
|
||||
.Where(x => x.Code == code)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (lastRecord == null) continue;
|
||||
|
||||
var processedRecord = CreateProcessedRecord(lastRecord);
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}",
|
||||
ProcessorType, code, lastRecord.CreatedAt);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private Record CreateProcessedRecord(Record lastRecord)
|
||||
{
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = lastRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Copy all values from positions 1-32
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var value = ProcessHelper.GetValue(lastRecord, i);
|
||||
if (value != null)
|
||||
{
|
||||
ProcessHelper.SetValue(processedRecord, i, value);
|
||||
}
|
||||
}
|
||||
|
||||
processedRecord.Desc1 = lastRecord.Desc1;
|
||||
|
||||
return processedRecord;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user