377 lines
14 KiB
C#
377 lines
14 KiB
C#
|
using System.Globalization;
|
|||
|
|
using System.Text.RegularExpressions;
|
|||
|
|
using DiunaBI.Core.Services;
|
|||
|
|
using Google.Apis.Sheets.v4;
|
|||
|
|
using Google.Apis.Sheets.v4.Data;
|
|||
|
|
using Microsoft.EntityFrameworkCore;
|
|||
|
|
using DiunaBI.Core.Models;
|
|||
|
|
using DiunaBI.Core.Database.Context;
|
|||
|
|
using Microsoft.Extensions.Logging;
|
|||
|
|
|
|||
|
|
namespace DiunaBI.Plugins.Morska.Processors;
|
|||
|
|
|
|||
|
|
public class MorskaD6Processor : MorskaBaseProcessor
|
|||
|
|
{
|
|||
|
|
public override string ProcessorType => "Morska.Process.D6";
|
|||
|
|
|
|||
|
|
private readonly AppDbContext _db;
|
|||
|
|
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
|||
|
|
private readonly ILogger<MorskaT1R3Processor> _logger;
|
|||
|
|
|
|||
|
|
// Configuration properties loaded from layer records
|
|||
|
|
private int Year { get; set; }
|
|||
|
|
private string? CostSource { get; set; }
|
|||
|
|
private string? SellSource { get; set; }
|
|||
|
|
|
|||
|
|
public MorskaD6Processor(
|
|||
|
|
AppDbContext db,
|
|||
|
|
SpreadsheetsResource.ValuesResource googleSheetValues,
|
|||
|
|
ILogger<MorskaT1R3Processor> logger)
|
|||
|
|
{
|
|||
|
|
_db = db;
|
|||
|
|
_googleSheetValues = googleSheetValues;
|
|||
|
|
_logger = logger;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public override void Process(Layer processWorker)
|
|||
|
|
{
|
|||
|
|
try
|
|||
|
|
{
|
|||
|
|
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
|||
|
|
ProcessorType, processWorker.Name, processWorker.Id);
|
|||
|
|
|
|||
|
|
// Load configuration from layer records
|
|||
|
|
LoadConfiguration(processWorker);
|
|||
|
|
|
|||
|
|
// Validate required configuration
|
|||
|
|
ValidateConfiguration();
|
|||
|
|
|
|||
|
|
// Perform the actual processing
|
|||
|
|
PerformProcessing(processWorker);
|
|||
|
|
|
|||
|
|
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
|||
|
|
ProcessorType, processWorker.Name);
|
|||
|
|
}
|
|||
|
|
catch (Exception e)
|
|||
|
|
{
|
|||
|
|
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
|||
|
|
ProcessorType, processWorker.Name, processWorker.Id);
|
|||
|
|
throw;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void LoadConfiguration(Layer processWorker)
|
|||
|
|
{
|
|||
|
|
if (processWorker.Records == null)
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException("ProcessWorker has no records");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
|||
|
|
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException("Year record not found or invalid");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
Year = year;
|
|||
|
|
|
|||
|
|
CostSource = GetRecordValue(processWorker.Records, "SourceLayerCost");
|
|||
|
|
if (string.IsNullOrEmpty(CostSource))
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException("SourceLayerCost record not found");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
SellSource = GetRecordValue(processWorker.Records, "SourceLayerSell");
|
|||
|
|
if (string.IsNullOrEmpty(SellSource))
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException("SourceLayerCosts record not found");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogDebug(
|
|||
|
|
"{ProcessorType}: Configuration loaded - Year: {Year}, SourceCost: {CostSource}, SourceSell: {SellSource}",
|
|||
|
|
ProcessorType, Year, CostSource, SellSource);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void ValidateConfiguration()
|
|||
|
|
{
|
|||
|
|
var errors = new List<string>();
|
|||
|
|
|
|||
|
|
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
|||
|
|
if (string.IsNullOrEmpty(CostSource)) errors.Add("CostSource is required");
|
|||
|
|
if (string.IsNullOrEmpty(SellSource)) errors.Add("SellSource is required");
|
|||
|
|
|
|||
|
|
if (errors.Any())
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void PerformProcessing(Layer processWorker)
|
|||
|
|
{
|
|||
|
|
_logger.LogDebug(
|
|||
|
|
"{ProcessorType}: Processing data for Year: {Year}, CostSource: {CostSource}, SellSource: {SellSource}",
|
|||
|
|
ProcessorType, Year, CostSource, SellSource);
|
|||
|
|
|
|||
|
|
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
|||
|
|
|
|||
|
|
var dataSources = GetDataSources();
|
|||
|
|
|
|||
|
|
var newRecords = ProcessRecords(dataSources);
|
|||
|
|
|
|||
|
|
SaveProcessedLayer(processedLayer, newRecords);
|
|||
|
|
|
|||
|
|
_logger.LogInformation(
|
|||
|
|
"{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
|||
|
|
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
|||
|
|
{
|
|||
|
|
var processedLayer = _db.Layers
|
|||
|
|
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
|||
|
|
.OrderByDescending(x => x.CreatedAt)
|
|||
|
|
.FirstOrDefault();
|
|||
|
|
|
|||
|
|
if (processedLayer == null)
|
|||
|
|
{
|
|||
|
|
processedLayer = new Layer
|
|||
|
|
{
|
|||
|
|
Id = Guid.NewGuid(),
|
|||
|
|
Type = LayerType.Processed,
|
|||
|
|
ParentId = processWorker.Id,
|
|||
|
|
Number = _db.Layers.Count() + 1,
|
|||
|
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|||
|
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
|||
|
|
CreatedAt = DateTime.UtcNow,
|
|||
|
|
ModifiedAt = DateTime.UtcNow
|
|||
|
|
};
|
|||
|
|
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-D6";
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
|||
|
|
ProcessorType, processedLayer.Name);
|
|||
|
|
}
|
|||
|
|
else
|
|||
|
|
{
|
|||
|
|
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|||
|
|
processedLayer.ModifiedAt = DateTime.UtcNow;
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
|||
|
|
ProcessorType, processedLayer.Name);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return processedLayer;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private List<Layer> GetDataSources()
|
|||
|
|
{
|
|||
|
|
var costDataSource = _db.Layers
|
|||
|
|
.Include(layer => layer.Records!)
|
|||
|
|
.AsNoTracking()
|
|||
|
|
.FirstOrDefault(x => x.Name == CostSource && !x.IsDeleted && !x.IsCancelled);
|
|||
|
|
|
|||
|
|
if (costDataSource == null)
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException($"CostDataSource not found {CostSource}");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var sellDataSource = _db.Layers
|
|||
|
|
.Include(layer => layer.Records!)
|
|||
|
|
.AsNoTracking()
|
|||
|
|
.FirstOrDefault(x => x.Name == SellSource && !x.IsDeleted && !x.IsCancelled);
|
|||
|
|
|
|||
|
|
if (sellDataSource == null)
|
|||
|
|
{
|
|||
|
|
throw new InvalidOperationException($"SellDataSource not found {SellSource}");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Found both data sources data sources: {CostSource}, {SellSource}",
|
|||
|
|
ProcessorType, CostSource, SellSource);
|
|||
|
|
;
|
|||
|
|
|
|||
|
|
return [costDataSource, sellDataSource];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private List<Record> ProcessRecords(List<Layer> dataSources)
|
|||
|
|
{
|
|||
|
|
var newRecords = new List<Record>();
|
|||
|
|
|
|||
|
|
// L8542-D-DEPARTMENTS
|
|||
|
|
var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542);
|
|||
|
|
|
|||
|
|
var departmentLookup = new Dictionary<string, string>();
|
|||
|
|
if (dictionary?.Records != null)
|
|||
|
|
{
|
|||
|
|
foreach (var dictRecord in dictionary.Records)
|
|||
|
|
{
|
|||
|
|
if (!string.IsNullOrEmpty(dictRecord.Desc1) && !string.IsNullOrEmpty(dictRecord.Code))
|
|||
|
|
{
|
|||
|
|
departmentLookup[dictRecord.Desc1] = dictRecord.Code;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Loaded {DictCount} department mappings from dictionary",
|
|||
|
|
ProcessorType, departmentLookup.Count);
|
|||
|
|
}
|
|||
|
|
else
|
|||
|
|
{
|
|||
|
|
_logger.LogWarning("{ProcessorType}: Department dictionary (layer 8542) not found or has no records",
|
|||
|
|
ProcessorType);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var firstDataSource = dataSources.First();
|
|||
|
|
|
|||
|
|
if (firstDataSource.Records == null || !firstDataSource.Records.Any())
|
|||
|
|
{
|
|||
|
|
_logger.LogWarning("{ProcessorType}: First data source has no records to process", ProcessorType);
|
|||
|
|
return newRecords;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
var groupedData = firstDataSource.Records
|
|||
|
|
.Where(record => record.Code != null && record.Code.Length >= 8 && record.Value1.HasValue)
|
|||
|
|
.Select(record => new
|
|||
|
|
{
|
|||
|
|
Month = record.Code!.Substring(4, 2),
|
|||
|
|
Type = record.Code.Substring(6, 2),
|
|||
|
|
OriginalDepartment = record.Desc1 ?? string.Empty,
|
|||
|
|
Value = record.Value1!.Value,
|
|||
|
|
Department = GetDepartmentByType(record.Code.Substring(6, 2), record.Desc1 ?? string.Empty)
|
|||
|
|
})
|
|||
|
|
.Select(x => new
|
|||
|
|
{
|
|||
|
|
x.Month,
|
|||
|
|
x.Type,
|
|||
|
|
x.Value,
|
|||
|
|
x.Department,
|
|||
|
|
DepartmentCode = departmentLookup.TryGetValue(x.Department, out var value1)
|
|||
|
|
? value1
|
|||
|
|
: x.Department,
|
|||
|
|
FinalCode =
|
|||
|
|
$"2{(departmentLookup.TryGetValue(x.Department, out var value) ? value : x.Department)}{x.Type}{x.Month}"
|
|||
|
|
})
|
|||
|
|
.GroupBy(x => x.FinalCode)
|
|||
|
|
.Select(group => new
|
|||
|
|
{
|
|||
|
|
FinalCode = group.Key,
|
|||
|
|
Department = group.First().Department,
|
|||
|
|
DepartmentCode = group.First().DepartmentCode,
|
|||
|
|
Type = group.First().Type,
|
|||
|
|
Month = group.First().Month,
|
|||
|
|
TotalValue = group.Sum(x => x.Value)
|
|||
|
|
})
|
|||
|
|
.ToList();
|
|||
|
|
|
|||
|
|
foreach (var groupedRecord in groupedData)
|
|||
|
|
{
|
|||
|
|
var newRecord = new Record
|
|||
|
|
{
|
|||
|
|
Id = Guid.NewGuid(),
|
|||
|
|
Code = groupedRecord.FinalCode,
|
|||
|
|
CreatedAt = DateTime.UtcNow,
|
|||
|
|
ModifiedAt = DateTime.UtcNow,
|
|||
|
|
Value1 = groupedRecord.TotalValue,
|
|||
|
|
Desc1 = groupedRecord.Department
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
newRecords.Add(newRecord);
|
|||
|
|
|
|||
|
|
_logger.LogDebug(
|
|||
|
|
"{ProcessorType}: Created summed record {NewRecordCode} with total value {Value} for department {Department} (code: {DepartmentCode}), type {Type}, month {Month}",
|
|||
|
|
ProcessorType, newRecord.Code, newRecord.Value1, groupedRecord.Department, groupedRecord.DepartmentCode,
|
|||
|
|
groupedRecord.Type, groupedRecord.Month);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogInformation(
|
|||
|
|
"{ProcessorType}: Processed {GroupCount} unique grouped records from {OriginalCount} original records",
|
|||
|
|
ProcessorType, newRecords.Count, firstDataSource.Records.Count);
|
|||
|
|
|
|||
|
|
return newRecords;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
|||
|
|
{
|
|||
|
|
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
|||
|
|
|
|||
|
|
if (!existsInDb)
|
|||
|
|
{
|
|||
|
|
_db.Layers.Add(processedLayer);
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
|||
|
|
}
|
|||
|
|
else
|
|||
|
|
{
|
|||
|
|
_db.Layers.Update(processedLayer);
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
SaveRecords(processedLayer.Id, newRecords);
|
|||
|
|
_db.SaveChanges();
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
|||
|
|
ProcessorType, newRecords.Count, processedLayer.Id);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
|||
|
|
{
|
|||
|
|
// Remove existing records for this layer
|
|||
|
|
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
|||
|
|
if (toDelete.Count > 0)
|
|||
|
|
{
|
|||
|
|
_db.Records.RemoveRange(toDelete);
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
|||
|
|
ProcessorType, toDelete.Count, layerId);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// Add new records
|
|||
|
|
foreach (var record in records)
|
|||
|
|
{
|
|||
|
|
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|||
|
|
record.CreatedAt = DateTime.UtcNow;
|
|||
|
|
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
|||
|
|
record.ModifiedAt = DateTime.UtcNow;
|
|||
|
|
record.LayerId = layerId;
|
|||
|
|
_db.Records.Add(record);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
|||
|
|
ProcessorType, records.Count, layerId);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private string? GetRecordValue(ICollection<Record> records, string code)
|
|||
|
|
{
|
|||
|
|
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private string GetDepartmentByType(string type, string originalDepartment)
|
|||
|
|
{
|
|||
|
|
var typesThatUseDepartment = new[] { "02", "09", "10", "11", "12", "13", "14", "15" };
|
|||
|
|
var typesThatUseAK = new[] { "03", "06", "07", "08" };
|
|||
|
|
|
|||
|
|
if (typesThatUseDepartment.Contains(type))
|
|||
|
|
{
|
|||
|
|
return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (typesThatUseAK.Contains(type))
|
|||
|
|
{
|
|||
|
|
return "AK";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (type == "04")
|
|||
|
|
{
|
|||
|
|
return "PU";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (type == "05")
|
|||
|
|
{
|
|||
|
|
return "OTHER";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
{
|
|||
|
|
_logger.LogWarning("{ProcessorType}: Unknown type {Type}, using {Department}",
|
|||
|
|
nameof(MorskaD6Processor), type,
|
|||
|
|
string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment);
|
|||
|
|
return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|