after refactor cleanup
This commit is contained in:
17
DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj
Normal file
17
DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj
Normal file
@@ -0,0 +1,17 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="10.0.0" />
|
||||
<PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3525" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\DiunaBI.Domain\DiunaBI.Domain.csproj" />
|
||||
<ProjectReference Include="..\DiunaBI.Infrastructure\DiunaBI.Infrastructure.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
11
DiunaBI.Plugins.Morska/Exporters/MorskaBaseExporter.cs
Normal file
11
DiunaBI.Plugins.Morska/Exporters/MorskaBaseExporter.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Interfaces;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Exporters;
|
||||
|
||||
public abstract class MorskaBaseExporter : IDataExporter
|
||||
{
|
||||
public abstract string ExporterType { get; }
|
||||
public virtual bool CanExport(string exporterType) => ExporterType == exporterType;
|
||||
public abstract void Export(Layer layer);
|
||||
}
|
||||
125
DiunaBI.Plugins.Morska/Exporters/googleSheet.export.cs
Normal file
125
DiunaBI.Plugins.Morska/Exporters/googleSheet.export.cs
Normal file
@@ -0,0 +1,125 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Exporters;
|
||||
|
||||
public class GoogleSheetExport : MorskaBaseExporter
|
||||
{
|
||||
public override string ExporterType => "GoogleSheet";
|
||||
private readonly GoogleDriveHelper _googleDriveHelper;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly IConfiguration _configuration;
|
||||
public GoogleSheetExport(
|
||||
GoogleDriveHelper googleDriveHelper,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
_googleDriveHelper = googleDriveHelper;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_configuration = configuration;
|
||||
}
|
||||
public override void Export(Layer layer)
|
||||
{
|
||||
if (_googleDriveHelper.Service is null)
|
||||
{
|
||||
throw new Exception("Google Drive API not initialized");
|
||||
}
|
||||
try
|
||||
{
|
||||
|
||||
var data = new List<IList<object>> { new List<object> { layer.Name! } };
|
||||
|
||||
switch (layer.Type)
|
||||
{
|
||||
case LayerType.Import:
|
||||
{
|
||||
data.Add(new List<object> { "Code", "Value1" });
|
||||
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Value1! }));
|
||||
break;
|
||||
}
|
||||
case LayerType.Administration:
|
||||
{
|
||||
data.Add(new List<object> { "Code", "Desc1" });
|
||||
data.AddRange(layer.Records!.Select(record => new List<object> { record.Code!, record.Desc1! }));
|
||||
break;
|
||||
}
|
||||
case LayerType.Processed:
|
||||
{
|
||||
data.Add(new List<object> { "Code", "Value1", "Value2", "Value3", "Value3",
|
||||
"Value5", "Value6", "Value7", "Value8", "Value9", "Value10",
|
||||
"Value11", "Value12", "Value13", "Value14", "Value15", "Value16",
|
||||
"Value17", "Value18", "Value19", "Value20", "Value21", "Value22",
|
||||
"Value23", "Value24", "Value25", "Value26", "Value27", "Value28",
|
||||
"Value29", "Value30", "Value31", "Value32"});
|
||||
|
||||
data.AddRange(layer.Records!.Select(record => new List<object>
|
||||
{
|
||||
record.Code!,
|
||||
record.Value1!,
|
||||
record.Value2!,
|
||||
record.Value3!,
|
||||
record.Value4!,
|
||||
record.Value5!,
|
||||
record.Value6!,
|
||||
record.Value7!,
|
||||
record.Value8!,
|
||||
record.Value9!,
|
||||
record.Value10!,
|
||||
record.Value11!,
|
||||
record.Value12!,
|
||||
record.Value13!,
|
||||
record.Value14!,
|
||||
record.Value15!,
|
||||
record.Value16!,
|
||||
record.Value17!,
|
||||
record.Value18!,
|
||||
record.Value19!,
|
||||
record.Value20!,
|
||||
record.Value21!,
|
||||
record.Value22!,
|
||||
record.Value23!,
|
||||
record.Value24!,
|
||||
record.Value25!,
|
||||
record.Value26!,
|
||||
record.Value27!,
|
||||
record.Value28!,
|
||||
record.Value29!,
|
||||
record.Value30!,
|
||||
record.Value31!,
|
||||
record.Value32!
|
||||
}));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Exception("Wrong LayerType");
|
||||
}
|
||||
|
||||
var body = new Google.Apis.Drive.v3.Data.File
|
||||
{
|
||||
Name = $"{DateTime.Now.ToString(new CultureInfo("pl-PL"))}",
|
||||
MimeType = "application/vnd.google-apps.spreadsheet",
|
||||
Parents = new List<string?> { _configuration["exportDirectory"] }
|
||||
};
|
||||
var request = _googleDriveHelper.Service.Files.Create(body);
|
||||
var file = request.Execute();
|
||||
|
||||
var sheetId = file.Id;
|
||||
var range = $"Sheet1!A1:AG${data.Count}";
|
||||
|
||||
var valueRange = new ValueRange { Values = data };
|
||||
|
||||
var updateRequest = _googleSheetValues.Update(valueRange, sheetId, range);
|
||||
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
|
||||
updateRequest.Execute();
|
||||
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
13
DiunaBI.Plugins.Morska/Importers/MorskaBaseImporter.cs
Normal file
13
DiunaBI.Plugins.Morska/Importers/MorskaBaseImporter.cs
Normal file
@@ -0,0 +1,13 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Interfaces;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public abstract class MorskaBaseImporter : IDataImporter
|
||||
{
|
||||
public abstract string ImporterType { get; }
|
||||
|
||||
public virtual bool CanImport(string importerType) => ImporterType == importerType;
|
||||
|
||||
public abstract void Import(Layer importWorker);
|
||||
}
|
||||
415
DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
Normal file
415
DiunaBI.Plugins.Morska/Importers/MorskaD1Importer.cs
Normal file
@@ -0,0 +1,415 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaD1Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.D1";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaD1Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cache for sheet data
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaD1Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaD1Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, need to create new layer",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count < 2)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var isUpToDate = true;
|
||||
|
||||
foreach (var row in data)
|
||||
{
|
||||
if (row.Count <= 1 || string.IsNullOrEmpty(row[0]?.ToString())) continue;
|
||||
|
||||
var code = row[0].ToString();
|
||||
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||
|
||||
if (record == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||
ImporterType, code, importWorker.Name);
|
||||
isUpToDate = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check values 3-17 (Value1-Value15)
|
||||
for (int i = 3; i <= 17 && i < row.Count; i++)
|
||||
{
|
||||
var sheetValue = ParseValue(row[i]?.ToString());
|
||||
var dbValue = GetRecordValueByIndex(record, i - 2); // Value1 starts at index 3
|
||||
|
||||
if (Math.Abs((sheetValue ?? 0) - (dbValue ?? 0)) >= 0.01)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code} at position {Position}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, i - 2, dbValue, sheetValue);
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
// check also Desc1 from records (Value18 in GSheet)
|
||||
if (row.Count <= 18) continue;
|
||||
var sheetDesc1 = row[18]?.ToString();
|
||||
var recordDesc1 = record.Desc1;
|
||||
|
||||
var normalizedSheetDesc1 = string.IsNullOrEmpty(sheetDesc1) ? null : sheetDesc1;
|
||||
var normalizedRecordDesc1 = string.IsNullOrEmpty(recordDesc1) ? null : recordDesc1;
|
||||
|
||||
if (normalizedSheetDesc1 == normalizedRecordDesc1) continue;
|
||||
_logger.LogDebug("{ImporterType}: Desc1 mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, normalizedRecordDesc1, normalizedSheetDesc1);
|
||||
isUpToDate = false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
var newRecords = (from t in data
|
||||
where t.Count > 1 && !string.IsNullOrEmpty(t[0]?.ToString())
|
||||
select new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = t[0].ToString(),
|
||||
Value1 = IndexExists(t, 3) ? ParseValue(t[3]?.ToString()) : null,
|
||||
Value2 = IndexExists(t, 4) ? ParseValue(t[4]?.ToString()) : null,
|
||||
Value3 = IndexExists(t, 5) ? ParseValue(t[5]?.ToString()) : null,
|
||||
Value4 = IndexExists(t, 6) ? ParseValue(t[6]?.ToString()) : null,
|
||||
Value5 = IndexExists(t, 7) ? ParseValue(t[7]?.ToString()) : null,
|
||||
Value6 = IndexExists(t, 8) ? ParseValue(t[8]?.ToString()) : null,
|
||||
Value7 = IndexExists(t, 9) ? ParseValue(t[9]?.ToString()) : null,
|
||||
Value8 = IndexExists(t, 10) ? ParseValue(t[10]?.ToString()) : null,
|
||||
Value9 = IndexExists(t, 11) ? ParseValue(t[11]?.ToString()) : null,
|
||||
Value10 = IndexExists(t, 12) ? ParseValue(t[12]?.ToString()) : null,
|
||||
Value11 = IndexExists(t, 13) ? ParseValue(t[13]?.ToString()) : null,
|
||||
Value12 = IndexExists(t, 14) ? ParseValue(t[14]?.ToString()) : null,
|
||||
Value13 = IndexExists(t, 15) ? ParseValue(t[15]?.ToString()) : null,
|
||||
Value14 = IndexExists(t, 16) ? ParseValue(t[16]?.ToString()) : null,
|
||||
Value15 = IndexExists(t, 17) ? ParseValue(t[17]?.ToString()) : null,
|
||||
Desc1 = IndexExists(t, 18) ? t[18]?.ToString() : null,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
}).ToList();
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private double? ParseValue(string? value)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value) || value == "#DIV/0!") return null;
|
||||
value = new string(value.Where(c => char.IsDigit(c) || c == '.' || c == ',' || c == '-').ToArray());
|
||||
try
|
||||
{
|
||||
double.TryParse(value, CultureInfo.GetCultureInfo("pl-PL"), out var result);
|
||||
return result;
|
||||
}
|
||||
catch (FormatException e)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Failed to parse value '{Value}': {Error}",
|
||||
ImporterType, value, e.Message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private bool IndexExists(IList<object> array, int index)
|
||||
{
|
||||
return array != null && index >= 0 && index < array.Count;
|
||||
}
|
||||
|
||||
private double? GetRecordValueByIndex(Record record, int valueIndex)
|
||||
{
|
||||
return valueIndex switch
|
||||
{
|
||||
1 => record.Value1,
|
||||
2 => record.Value2,
|
||||
3 => record.Value3,
|
||||
4 => record.Value4,
|
||||
5 => record.Value5,
|
||||
6 => record.Value6,
|
||||
7 => record.Value7,
|
||||
8 => record.Value8,
|
||||
9 => record.Value9,
|
||||
10 => record.Value10,
|
||||
11 => record.Value11,
|
||||
12 => record.Value12,
|
||||
13 => record.Value13,
|
||||
14 => record.Value14,
|
||||
15 => record.Value15,
|
||||
_ => null
|
||||
};
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
428
DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs
Normal file
428
DiunaBI.Plugins.Morska/Importers/MorskaD3Importer.cs
Normal file
@@ -0,0 +1,428 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaD3Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.D3";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaD3Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private string? ImportType { get; set; }
|
||||
private string? SheetId { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cached deserialized data
|
||||
private List<Record>? _cachedRecords;
|
||||
private string? _cachedDataInboxId;
|
||||
|
||||
public MorskaD3Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaD3Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Clear cache at start
|
||||
_cachedRecords = null;
|
||||
_cachedDataInboxId = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
// Deserialize data early - right after LoadConfiguration
|
||||
DeserializeDataInboxData();
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
// Export to Google Sheets after successful import
|
||||
ExportToGoogleSheets();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clear cache after import
|
||||
_cachedRecords = null;
|
||||
_cachedDataInboxId = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
ImportType = GetRecordValue(importWorker.Records, "ImportType");
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ImporterType}: Configuration loaded for {ImportWorkerName} - Type: {ImportType}, SheetId: {SheetId}",
|
||||
ImporterType, importWorker.Name, ImportType, SheetId);
|
||||
}
|
||||
|
||||
private void DeserializeDataInboxData()
|
||||
{
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
throw new InvalidOperationException($"DataInbox not found for type: {ImportType}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Found DataInbox {DataInboxId}, created at {CreatedAt}",
|
||||
ImporterType, dataInbox.Id, dataInbox.CreatedAt);
|
||||
|
||||
try
|
||||
{
|
||||
var data = Convert.FromBase64String(dataInbox.Data);
|
||||
var jsonString = Encoding.UTF8.GetString(data);
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Decoded {DataSize} bytes from base64",
|
||||
ImporterType, data.Length);
|
||||
|
||||
var records = JsonSerializer.Deserialize<List<Record>>(jsonString);
|
||||
if (records == null)
|
||||
{
|
||||
throw new InvalidOperationException($"DataInbox.Data is empty for: {dataInbox.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Deserialized {RecordCount} records from JSON",
|
||||
ImporterType, records.Count);
|
||||
|
||||
// Cache the deserialized data
|
||||
_cachedRecords = records;
|
||||
_cachedDataInboxId = dataInbox.Id.ToString();
|
||||
}
|
||||
catch (FormatException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid base64 data in DataInbox {DataInboxName}",
|
||||
ImporterType, ImportType);
|
||||
throw new InvalidOperationException($"Invalid base64 data in DataInbox: {ImportType}", e);
|
||||
}
|
||||
catch (JsonException e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Invalid JSON data in DataInbox {DataInboxName}",
|
||||
ImporterType, ImportType);
|
||||
throw new InvalidOperationException($"Invalid JSON data in DataInbox: {ImportType}", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Record> GetFilteredRecords()
|
||||
{
|
||||
if (_cachedRecords == null)
|
||||
{
|
||||
throw new InvalidOperationException("Data has not been deserialized yet");
|
||||
}
|
||||
|
||||
var filteredRecords = _cachedRecords.Where(x => x.Code!.StartsWith($"{ImportYear}{ImportMonth}")).ToList();
|
||||
if (filteredRecords.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No records found for period: {ImportYear}{ImportMonth}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Filtered to {FilteredCount} records for period {Year}{Month}",
|
||||
ImporterType, filteredRecords.Count, ImportYear, ImportMonth);
|
||||
|
||||
return filteredRecords;
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
var dataInbox = _db.DataInbox.OrderByDescending(x => x.CreatedAt).FirstOrDefault(x => x.Name == ImportType);
|
||||
if (dataInbox == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No DataInbox found for type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if imported layer is up to date
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
if (string.IsNullOrEmpty(ImportType)) errors.Add("ImportType is required");
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x =>
|
||||
x.ParentId == importWorker.Id
|
||||
&& x.Name!.Contains($"I-{ImportName}-{ImportYear}/{ImportMonth}"))
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, import needed",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var currentRecords = GetFilteredRecords();
|
||||
|
||||
// Compare record counts first
|
||||
if (newestLayer.Records?.Count != currentRecords.Count)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Record count mismatch - DB: {DbCount}, DataInbox: {DataCount}",
|
||||
ImporterType, newestLayer.Records?.Count ?? 0, currentRecords.Count);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare individual records
|
||||
foreach (var currentRecord in currentRecords)
|
||||
{
|
||||
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == currentRecord.Code);
|
||||
if (existingRecord == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Record with code {Code} not found in existing layer",
|
||||
ImporterType, currentRecord.Code);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compare all relevant fields
|
||||
if (Math.Abs((existingRecord.Value1 ?? 0) - (currentRecord.Value1 ?? 0)) > 0.001 ||
|
||||
existingRecord.Desc1 != currentRecord.Desc1)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Data difference found for code {Code}",
|
||||
ImporterType, currentRecord.Code);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: All records match, layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Starting import for DataInbox type {ImportType}",
|
||||
ImporterType, ImportType);
|
||||
|
||||
var filteredRecords = GetFilteredRecords();
|
||||
|
||||
// Prepare records for saving
|
||||
var recordsToSave = filteredRecords.Select(x =>
|
||||
{
|
||||
x.Id = Guid.NewGuid();
|
||||
x.CreatedAt = DateTime.UtcNow;
|
||||
x.ModifiedAt = DateTime.UtcNow;
|
||||
return x;
|
||||
}).ToList();
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, recordsToSave);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation(
|
||||
"{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, recordsToSave.Count, layer.Name, layer.Id);
|
||||
}
|
||||
|
||||
private void ExportToGoogleSheets()
|
||||
{
|
||||
if (string.IsNullOrEmpty(SheetId))
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: SheetId not configured, skipping Google Sheets export",
|
||||
ImporterType);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Starting Google Sheets export to {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
|
||||
var filteredRecords = GetFilteredRecords();
|
||||
var sheetTabName = ProcessHelper.GetSheetName(int.Parse(ImportMonth!), int.Parse(ImportYear!));
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using sheet tab name: {SheetTabName}",
|
||||
ImporterType, sheetTabName);
|
||||
|
||||
// Get current sheet data
|
||||
var currentSheetData = _googleSheetValues.Get(SheetId!, $"{sheetTabName}!C7:D200").Execute();
|
||||
|
||||
var updateRequests = new List<ValueRange>();
|
||||
|
||||
for (var rowIndex = 0; rowIndex < 194; rowIndex++)
|
||||
{
|
||||
if (currentSheetData.Values == null || rowIndex >= currentSheetData.Values.Count)
|
||||
continue;
|
||||
|
||||
var existingRow = currentSheetData.Values[rowIndex];
|
||||
if (existingRow.Count == 0 || string.IsNullOrEmpty(existingRow[0]?.ToString()))
|
||||
continue;
|
||||
|
||||
var accountCode = existingRow[0].ToString()!;
|
||||
var matchingRecord = filteredRecords.FirstOrDefault(x => x.Desc1 == accountCode);
|
||||
if (matchingRecord == null)
|
||||
continue;
|
||||
|
||||
var newValue = matchingRecord.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0";
|
||||
var existingValue = (existingRow.Count > 1 ? existingRow[1] : "")?.ToString();
|
||||
|
||||
if (existingValue != newValue)
|
||||
{
|
||||
var range = $"{sheetTabName}!D{rowIndex + 7}";
|
||||
updateRequests.Add(new ValueRange
|
||||
{
|
||||
Range = range,
|
||||
Values = new List<IList<object>> { new List<object> { newValue } }
|
||||
});
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Scheduled update for {AccountCode} at {Range} with value {Value}",
|
||||
ImporterType, accountCode, range, newValue);
|
||||
}
|
||||
}
|
||||
|
||||
if (updateRequests.Count > 0)
|
||||
{
|
||||
var batchUpdate = new BatchUpdateValuesRequest
|
||||
{
|
||||
Data = updateRequests,
|
||||
ValueInputOption = "USER_ENTERED"
|
||||
};
|
||||
|
||||
_googleSheetValues.BatchUpdate(batchUpdate, SheetId!).Execute();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Updated {Count} cells in Google Sheet {SheetId}",
|
||||
ImporterType, updateRequests.Count, SheetId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: No changes to export to Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to export to Google Sheets {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
397
DiunaBI.Plugins.Morska/Importers/MorskaFK2Importer.cs
Normal file
@@ -0,0 +1,397 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaFk2Importer : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.FK2";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaFk2Importer> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
// Cache for sheet data
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaFk2Importer(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaFk2Importer> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// ✅ Clear cache at start
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
ValidateConfiguration();
|
||||
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// ✅ Clear cache after import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ✅ Dodaj metodę cache
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var isUpToDate = true;
|
||||
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString())) continue;
|
||||
|
||||
try
|
||||
{
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3) continue;
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
var record = newestLayer.Records?.FirstOrDefault(x => x.Code == code);
|
||||
if (record == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Code {Code} not found in database for {ImportWorkerName}",
|
||||
ImporterType, code, importWorker.Name);
|
||||
isUpToDate = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (double.TryParse(data[i][9]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) &&
|
||||
Math.Abs((double)(record.Value1 - value)!) >= 0.01)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Value mismatch for code {Code}: DB={DbValue}, Sheet={SheetValue}",
|
||||
ImporterType, code, record.Value1, value);
|
||||
isUpToDate = false;
|
||||
}
|
||||
|
||||
if (record.Desc1 != data[i][3]?.ToString())
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Description mismatch for code {Code}: DB={DbDesc}, Sheet={SheetDesc}",
|
||||
ImporterType, code, record.Desc1, data[i][3]?.ToString());
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{ImporterType}: Error processing row {Index} for comparison",
|
||||
ImporterType, i);
|
||||
isUpToDate = false;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Layer {ImportWorkerName} is {Status}",
|
||||
ImporterType, importWorker.Name, isUpToDate ? "up to date" : "outdated");
|
||||
|
||||
return isUpToDate;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking if layer {ImportWorkerName} is up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
// ✅ Użyj cache zamiast bezpośredniego API
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null)
|
||||
{
|
||||
for (var i = 0; i < data.Count; i++)
|
||||
{
|
||||
if (data[i].Count <= 9 || string.IsNullOrEmpty(data[i][3]?.ToString()))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - insufficient columns or empty desc",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dateArr = data[i][1].ToString()!.Split(".");
|
||||
if (dateArr.Length != 3)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Invalid date format in row {Index}: {Date}",
|
||||
ImporterType, i, data[i][1]);
|
||||
throw new InvalidOperationException($"Invalid date in row {i}");
|
||||
}
|
||||
|
||||
var number = data[i][2].ToString()!;
|
||||
if (number.Length == 1) number = $"0{number}";
|
||||
var code = dateArr[2] + dateArr[1] + dateArr[0] + number;
|
||||
|
||||
if (string.IsNullOrEmpty(data[i][9]?.ToString()) ||
|
||||
!double.TryParse(data[i][9].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping row {Index} - empty or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = code,
|
||||
Desc1 = data[i][3].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
newRecords.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
}
|
||||
360
DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs
Normal file
360
DiunaBI.Plugins.Morska/Importers/MorskaStandardImporter.cs
Normal file
@@ -0,0 +1,360 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Importers;
|
||||
|
||||
public class MorskaStandardImporter : MorskaBaseImporter
|
||||
{
|
||||
public override string ImporterType => "Morska.Import.Standard";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaStandardImporter> _logger;
|
||||
|
||||
// Configuration properties
|
||||
private string? SheetId { get; set; }
|
||||
private string? SheetTabName { get; set; }
|
||||
private string? DataRange { get; set; }
|
||||
private string? ImportYear { get; set; }
|
||||
private string? ImportMonth { get; set; }
|
||||
private string? ImportName { get; set; }
|
||||
private DateTime? StartDate { get; set; }
|
||||
private DateTime? EndDate { get; set; }
|
||||
private bool IsEnabled { get; set; }
|
||||
|
||||
private IList<IList<object>>? _cachedSheetData;
|
||||
private string? _cachedDataKey;
|
||||
|
||||
public MorskaStandardImporter(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaStandardImporter> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Import(Layer importWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Starting import for {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
|
||||
// Clear cache before import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(importWorker);
|
||||
|
||||
// Check if import should be performed
|
||||
if (!ShouldPerformImport(importWorker))
|
||||
{
|
||||
_logger.LogInformation("{ImporterType}: Import not needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual import
|
||||
PerformImport(importWorker);
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully completed import for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Failed to import {ImportWorkerName} ({ImportWorkerId})",
|
||||
ImporterType, importWorker.Name, importWorker.Id);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Clear cache after import
|
||||
_cachedSheetData = null;
|
||||
_cachedDataKey = null;
|
||||
}
|
||||
}
|
||||
|
||||
private IList<IList<object>>? GetSheetData()
|
||||
{
|
||||
var currentDataKey = $"{SheetId}#{SheetTabName}#{DataRange}";
|
||||
|
||||
if (_cachedSheetData != null && _cachedDataKey == currentDataKey)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Using cached sheet data for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
return _cachedSheetData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Fetching data from Google Sheets API for {DataKey}",
|
||||
ImporterType, currentDataKey);
|
||||
|
||||
var dataRangeResponse = _googleSheetValues.Get(SheetId!, $"{SheetTabName}!{DataRange}").Execute();
|
||||
_cachedSheetData = dataRangeResponse.Values;
|
||||
_cachedDataKey = currentDataKey;
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Cached {RowCount} rows from Google Sheet",
|
||||
ImporterType, _cachedSheetData?.Count ?? 0);
|
||||
|
||||
return _cachedSheetData;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error fetching data from Google Sheet {SheetId}",
|
||||
ImporterType, SheetId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer importWorker)
|
||||
{
|
||||
if (importWorker.Records == null) return;
|
||||
|
||||
SheetId = GetRecordValue(importWorker.Records, "SheetId");
|
||||
SheetTabName = GetRecordValue(importWorker.Records, "SheetTabName");
|
||||
DataRange = GetRecordValue(importWorker.Records, "DataRange");
|
||||
ImportYear = GetRecordValue(importWorker.Records, "ImportYear");
|
||||
ImportMonth = GetRecordValue(importWorker.Records, "ImportMonth");
|
||||
ImportName = GetRecordValue(importWorker.Records, "ImportName");
|
||||
IsEnabled = GetRecordValue(importWorker.Records, "IsEnabled") == "True";
|
||||
|
||||
var startDateStr = GetRecordValue(importWorker.Records, "StartDate");
|
||||
if (startDateStr != null && DateTime.TryParseExact(startDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var startDate))
|
||||
{
|
||||
StartDate = startDate;
|
||||
}
|
||||
|
||||
var endDateStr = GetRecordValue(importWorker.Records, "EndDate");
|
||||
if (endDateStr != null && DateTime.TryParseExact(endDateStr, "yyyy.MM.dd", null, DateTimeStyles.None, out var endDate))
|
||||
{
|
||||
EndDate = endDate;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Configuration loaded for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
}
|
||||
|
||||
private bool ShouldPerformImport(Layer importWorker)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Import disabled for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check date range if configured
|
||||
if (StartDate.HasValue && EndDate.HasValue)
|
||||
{
|
||||
var now = DateTime.UtcNow.Date;
|
||||
if (now >= StartDate.Value.Date && now <= EndDate.Value.Date)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Within date range, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Outside date range - check if imported layer is up to date
|
||||
if (!IsImportedLayerUpToDate(importWorker))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Outside date range but layer is out of date, import needed for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Outside date range and layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// No date constraints - always import
|
||||
return true;
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrEmpty(SheetId)) errors.Add("SheetId is required");
|
||||
if (string.IsNullOrEmpty(SheetTabName)) errors.Add("SheetTabName is required");
|
||||
if (string.IsNullOrEmpty(DataRange)) errors.Add("DataRange is required");
|
||||
if (string.IsNullOrEmpty(ImportYear)) errors.Add("ImportYear is required");
|
||||
if (string.IsNullOrEmpty(ImportMonth)) errors.Add("ImportMonth is required");
|
||||
if (string.IsNullOrEmpty(ImportName)) errors.Add("ImportName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
}
|
||||
|
||||
private bool IsImportedLayerUpToDate(Layer importWorker)
|
||||
{
|
||||
var newestLayer = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == importWorker.Id)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (newestLayer == null)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: No child layers found for {ImportWorkerName}, treating as up to date",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
if (data == null || data.Count < 2)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: No data found in sheet for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check if the number of columns matches
|
||||
if (data[0].Count != data[1].Count)
|
||||
{
|
||||
_logger.LogWarning("{ImporterType}: Column count mismatch in imported data for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var existingRecord = newestLayer.Records?.FirstOrDefault(x => x.Code == data[0][i].ToString());
|
||||
if (existingRecord == null || existingRecord.Value1 != value)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Imported data is newer or different for code {Code}",
|
||||
ImporterType, data[0][i]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error checking imported layer up-to-date status for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Imported layer is up to date for {ImportWorkerName}",
|
||||
ImporterType, importWorker.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void PerformImport(Layer importWorker)
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Importing from sheet {SheetId}, tab {SheetTabName}, range {DataRange}",
|
||||
ImporterType, SheetId, SheetTabName, DataRange);
|
||||
|
||||
var layer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Number = _db.Layers.Count() + 1,
|
||||
ParentId = importWorker.Id,
|
||||
Type = LayerType.Import,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
layer.Name = $"L{layer.Number}-I-{ImportName}-{ImportYear}/{ImportMonth}-{DateTime.Now:yyyyMMddHHmm}";
|
||||
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
try
|
||||
{
|
||||
var data = GetSheetData();
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Using data with {RowCount} rows from cache",
|
||||
ImporterType, data?.Count ?? 0);
|
||||
|
||||
if (data != null && data.Count >= 2)
|
||||
{
|
||||
for (var i = 0; i < data[1].Count; i++)
|
||||
{
|
||||
if (string.IsNullOrEmpty(data[0][i]?.ToString()) ||
|
||||
!double.TryParse(data[1][i]?.ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value))
|
||||
{
|
||||
_logger.LogDebug("{ImporterType}: Skipping column {Index} - empty code or invalid value",
|
||||
ImporterType, i);
|
||||
continue;
|
||||
}
|
||||
|
||||
var record = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = data[0][i].ToString(),
|
||||
Value1 = value,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
newRecords.Add(record);
|
||||
}
|
||||
}
|
||||
|
||||
_db.Layers.Add(layer);
|
||||
SaveRecords(layer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogInformation("{ImporterType}: Successfully imported {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ImporterType, newRecords.Count, layer.Name, layer.Id);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ImporterType}: Error importing data from cached sheet data", ImporterType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
}
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ImporterType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ImporterType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
13
DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs
Normal file
13
DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs
Normal file
@@ -0,0 +1,13 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Interfaces;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public abstract class MorskaBaseProcessor : IDataProcessor
|
||||
{
|
||||
public abstract string ProcessorType { get; }
|
||||
|
||||
public virtual bool CanProcess(string processorType) => ProcessorType == processorType;
|
||||
|
||||
public abstract void Process(Layer processWorker);
|
||||
}
|
||||
755
DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs
Normal file
755
DiunaBI.Plugins.Morska/Processors/MorskaD6Processor.cs
Normal file
@@ -0,0 +1,755 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services.Calculations;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaD6Processor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.D6";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT1R3Processor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private string? CostSource { get; set; }
|
||||
private string? SellSource { get; set; }
|
||||
private List<string> SellCodesConfiguration { get; set; } = new();
|
||||
|
||||
public MorskaD6Processor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT1R3Processor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
|
||||
Year = year;
|
||||
|
||||
CostSource = GetRecordValue(processWorker.Records, "SourceLayerCost");
|
||||
if (string.IsNullOrEmpty(CostSource))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayerCost record not found");
|
||||
}
|
||||
|
||||
SellSource = GetRecordValue(processWorker.Records, "SourceLayerSell");
|
||||
if (string.IsNullOrEmpty(SellSource))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayerCosts record not found");
|
||||
}
|
||||
|
||||
SellCodesConfiguration = processWorker.Records
|
||||
.Where(x => string.Equals(x.Code, "Sell-Code", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(x => x.Desc1!.Trim())
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
if (!SellCodesConfiguration.Any())
|
||||
{
|
||||
throw new InvalidOperationException("Sell-Code records not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ProcessorType}: Configuration loaded - Year: {Year}, SourceCost: {CostSource}, SourceSell: {SellSource}",
|
||||
ProcessorType, Year, CostSource, SellSource);
|
||||
}
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (string.IsNullOrEmpty(CostSource)) errors.Add("CostSource is required");
|
||||
if (string.IsNullOrEmpty(SellSource)) errors.Add("SellSource is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"{ProcessorType}: Processing data for Year: {Year}, CostSource: {CostSource}, SellSource: {SellSource}",
|
||||
ProcessorType, Year, CostSource, SellSource);
|
||||
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
UpdateGoogleSheetReport(processedLayer.Id);
|
||||
|
||||
_logger.LogInformation(
|
||||
"{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-D6";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var costDataSource = _db.Layers
|
||||
.Include(layer => layer.Records!)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault(x => x.Name == CostSource && !x.IsDeleted && !x.IsCancelled);
|
||||
|
||||
if (costDataSource == null)
|
||||
{
|
||||
throw new InvalidOperationException($"CostDataSource not found {CostSource}");
|
||||
}
|
||||
|
||||
var sellDataSource = _db.Layers
|
||||
.Include(layer => layer.Records!)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault(x => x.Name == SellSource && !x.IsDeleted && !x.IsCancelled);
|
||||
|
||||
if (sellDataSource == null)
|
||||
{
|
||||
throw new InvalidOperationException($"SellDataSource not found {SellSource}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found both data sources data sources: {CostSource}, {SellSource}",
|
||||
ProcessorType, CostSource, SellSource);
|
||||
;
|
||||
|
||||
return [costDataSource, sellDataSource];
|
||||
}
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
// L8542-D-DEPARTMENTS
|
||||
var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542);
|
||||
|
||||
var departmentLookup = new Dictionary<string, string>();
|
||||
if (dictionary?.Records != null)
|
||||
{
|
||||
foreach (var dictRecord in dictionary.Records)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(dictRecord.Desc1) && !string.IsNullOrEmpty(dictRecord.Code))
|
||||
{
|
||||
departmentLookup[dictRecord.Desc1] = dictRecord.Code;
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Loaded {DictCount} department mappings from dictionary",
|
||||
ProcessorType, departmentLookup.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Department dictionary (layer 8542) not found or has no records",
|
||||
ProcessorType);
|
||||
}
|
||||
|
||||
// COSTS
|
||||
|
||||
var firstDataSource = dataSources.First();
|
||||
|
||||
if (firstDataSource.Records == null || !firstDataSource.Records.Any())
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: First data source has no records to process", ProcessorType);
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
var groupedData = firstDataSource.Records
|
||||
.Where(record => record is { Code: { Length: >= 8 }, Value1: not null })
|
||||
.Select(record => new
|
||||
{
|
||||
Month = record.Code!.Substring(4, 2),
|
||||
Type = record.Code.Substring(6, 2),
|
||||
OriginalDepartment = record.Desc1 ?? string.Empty,
|
||||
Value = record.Value1!.Value,
|
||||
Department = GetDepartmentByType(record.Code.Substring(6, 2), record.Desc1 ?? string.Empty)
|
||||
})
|
||||
.Select(x => new
|
||||
{
|
||||
x.Month,
|
||||
x.Type,
|
||||
x.Value,
|
||||
x.Department,
|
||||
DepartmentCode = departmentLookup.TryGetValue(x.Department, out var value1)
|
||||
? value1
|
||||
: x.Department,
|
||||
FinalCode =
|
||||
$"2{(departmentLookup.TryGetValue(x.Department, out var value) ? value : x.Department)}{x.Type}{x.Month}"
|
||||
})
|
||||
.GroupBy(x => x.FinalCode)
|
||||
.Select(group => new
|
||||
{
|
||||
FinalCode = group.Key,
|
||||
Department = group.First().Department,
|
||||
DepartmentCode = group.First().DepartmentCode,
|
||||
Type = group.First().Type,
|
||||
Month = group.First().Month,
|
||||
TotalValue = group.Sum(x => x.Value)
|
||||
})
|
||||
.ToList();
|
||||
|
||||
foreach (var groupedRecord in groupedData)
|
||||
{
|
||||
// hack for 2206 ([2206]=[2206]+[2203]
|
||||
double value = groupedRecord.TotalValue;
|
||||
if (groupedRecord.FinalCode.StartsWith("2206"))
|
||||
{
|
||||
var month = groupedRecord.FinalCode.Substring(4, 2);
|
||||
var toSumUp = groupedData.FirstOrDefault(x => x.FinalCode == $"2203{month}");
|
||||
if (toSumUp == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: 2203{month} not found (to sum up with 2206{month}", ProcessorType, month, month);
|
||||
}
|
||||
else
|
||||
{
|
||||
value+= toSumUp.TotalValue;
|
||||
}
|
||||
}
|
||||
var newRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = groupedRecord.FinalCode,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = value,
|
||||
Desc1 = groupedRecord.Department
|
||||
};
|
||||
|
||||
newRecords.Add(newRecord);
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ProcessorType}: Created summed record {NewRecordCode} with total value {Value} for department {Department} (code: {DepartmentCode}), type {Type}, month {Month}",
|
||||
ProcessorType, newRecord.Code, newRecord.Value1, groupedRecord.Department, groupedRecord.DepartmentCode,
|
||||
groupedRecord.Type, groupedRecord.Month);
|
||||
}
|
||||
|
||||
// SELLS
|
||||
|
||||
var secondDataSource = dataSources.Last();
|
||||
|
||||
if (secondDataSource.Records == null || !secondDataSource.Records.Any())
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Second data source has no records to process", ProcessorType);
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
|
||||
foreach (var sellCodeConfig in SellCodesConfiguration)
|
||||
{
|
||||
var calc = new BaseCalc(sellCodeConfig);
|
||||
if (!calc.IsFormulaCorrect())
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Invalid formula: {SellCodeConfig}", ProcessorType, sellCodeConfig);
|
||||
continue;
|
||||
}
|
||||
var codes = calc.GetCodes();
|
||||
var resultCode = calc.GetResultCode();
|
||||
|
||||
for (var i = 1; i <= 12; i++)
|
||||
{
|
||||
var monthRecords = secondDataSource.Records
|
||||
.Where(x => x.Code is { Length: 6 } && x.Code.EndsWith($"{i:D2}") && x.Value1.HasValue)
|
||||
.ToList();
|
||||
if (monthRecords.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var ingredients = monthRecords.ToDictionary(x => x.Code!.Substring(0, 4), x => x.Value1!.Value);
|
||||
double result = 0;
|
||||
try
|
||||
{
|
||||
result = calc.Calculate(ingredients);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to calculate sell code {ResultCode} for month {Month}",
|
||||
ProcessorType, resultCode, i);
|
||||
}
|
||||
|
||||
var newRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{resultCode}{i:D2}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = result
|
||||
};
|
||||
newRecords.Add(newRecord);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
// Process records from secondDataSource
|
||||
foreach (var record in secondDataSource.Records)
|
||||
{
|
||||
if (string.IsNullOrEmpty(record.Code) || record.Code.Length < 6 || !record.Value1.HasValue)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract month from last two digits
|
||||
var monthStr = record.Code.Substring(record.Code.Length - 2);
|
||||
if (!int.TryParse(monthStr, out var month) || month < 1 || month > 13)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Invalid month in code {Code}", ProcessorType, record.Code);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract base code (without month)
|
||||
var baseCode = record.Code.Substring(0, record.Code.Length - 2);
|
||||
|
||||
// Check if we have mapping for this code
|
||||
if (mappingDictionary.TryGetValue(baseCode, out var targetCode))
|
||||
{
|
||||
// Create target code with month
|
||||
var targetCodeWithMonth = $"{targetCode}{monthStr}";
|
||||
|
||||
// Add/sum value
|
||||
if (sellResults.ContainsKey(targetCodeWithMonth))
|
||||
{
|
||||
sellResults[targetCodeWithMonth] += record.Value1.Value;
|
||||
_logger.LogDebug("{ProcessorType}: Added to existing record {TargetCode}: {Value} (total: {Total})",
|
||||
ProcessorType, targetCodeWithMonth, record.Value1.Value, sellResults[targetCodeWithMonth]);
|
||||
}
|
||||
else
|
||||
{
|
||||
sellResults[targetCodeWithMonth] = record.Value1.Value;
|
||||
_logger.LogDebug("{ProcessorType}: Created new record {TargetCode}: {Value}",
|
||||
ProcessorType, targetCodeWithMonth, record.Value1.Value);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: No mapping found for code {BaseCode}", ProcessorType, baseCode);
|
||||
}
|
||||
}
|
||||
|
||||
// Add results to newRecords
|
||||
foreach (var sellResult in sellResults)
|
||||
{
|
||||
var newRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = sellResult.Key,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = sellResult.Value
|
||||
};
|
||||
|
||||
newRecords.Add(newRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added sell record {Code} with value {Value}",
|
||||
ProcessorType, newRecord.Code, newRecord.Value1);
|
||||
}
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Processed {SellRecordCount} sell records from {OriginalSellCount} source records",
|
||||
ProcessorType, sellResults.Count, secondDataSource.Records.Count);
|
||||
|
||||
_logger.LogInformation(
|
||||
"{ProcessorType}: Processed {GroupCount} unique grouped records from {OriginalCount} original records",
|
||||
ProcessorType, newRecords.Count, firstDataSource.Records.Count);
|
||||
*/
|
||||
return newRecords;
|
||||
}
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
private string GetDepartmentByType(string type, string originalDepartment)
|
||||
{
|
||||
var typesThatUseDepartment = new[] { "02", "09", "10", "11", "12", "13", "14", "15" };
|
||||
var typesThatUseAK = new[] { "03", "06", "07", "08" };
|
||||
|
||||
if (typesThatUseDepartment.Contains(type))
|
||||
{
|
||||
return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
|
||||
}
|
||||
|
||||
if (typesThatUseAK.Contains(type))
|
||||
{
|
||||
return "AK";
|
||||
}
|
||||
|
||||
if (type == "04")
|
||||
{
|
||||
return "PU";
|
||||
}
|
||||
|
||||
if (type == "05")
|
||||
{
|
||||
return "OTHER";
|
||||
}
|
||||
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Unknown type {Type}, using {Department}",
|
||||
nameof(MorskaD6Processor), type,
|
||||
string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment);
|
||||
return string.IsNullOrEmpty(originalDepartment) ? "OTHER" : originalDepartment;
|
||||
}
|
||||
}
|
||||
|
||||
// Export to Google
|
||||
|
||||
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||
{
|
||||
const string googleSheetName = "Raport_R6_DRAFT_2025";
|
||||
try
|
||||
{
|
||||
const string sheetId = "19AljwrZRg2Rc5hagkfK3u8LIo3x9_GmFNnZEeOJ5g_g";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}",
|
||||
ProcessorType, googleSheetName);
|
||||
|
||||
// Get processed layer data
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.Id == sourceId)
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||
}
|
||||
|
||||
// Get codes from sheet header (row 4, columns C to AA)
|
||||
var codesResponse = _googleSheetValues.Get(sheetId, $"{googleSheetName}!C4:AD4").Execute();
|
||||
var codesRow = codesResponse.Values[0];
|
||||
|
||||
// Update data based on 6-digit codes from processedLayer
|
||||
UpdateMonthlyDataFromCodes(sheetId, codesRow, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// Update yearly summary data (row 20)
|
||||
UpdateYearlySummaryData(sheetId, codesRow, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// Update timestamps
|
||||
UpdateTimestamps(sheetId, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}",
|
||||
ProcessorType, googleSheetName);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}",
|
||||
ProcessorType, googleSheetName);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateYearlySummaryData(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||
{
|
||||
const string googleSheetName = "Raport_R6_DRAFT_2025";
|
||||
|
||||
if (processedLayer.Records == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: No records found in processed layer for yearly summary", ProcessorType);
|
||||
return;
|
||||
}
|
||||
|
||||
var summaryValues = new List<object>();
|
||||
|
||||
foreach (string fourDigitCode in codesRow)
|
||||
{
|
||||
// Calculate sum for all months (01-12) for this 4-digit code
|
||||
double yearlySum = 0;
|
||||
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var sixDigitCode = $"{fourDigitCode}{month:D2}";
|
||||
var record = processedLayer.Records.FirstOrDefault(x => x.Code == sixDigitCode);
|
||||
|
||||
if (record?.Value1.HasValue == true)
|
||||
{
|
||||
yearlySum += record.Value1.Value;
|
||||
}
|
||||
}
|
||||
|
||||
summaryValues.Add(yearlySum.ToString(CultureInfo.GetCultureInfo("pl-PL")));
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Calculated yearly sum for code {FourDigitCode}: {YearlySum}",
|
||||
ProcessorType, fourDigitCode, yearlySum);
|
||||
}
|
||||
|
||||
// Update row 20 with yearly sums
|
||||
var valueRange = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { summaryValues }
|
||||
};
|
||||
|
||||
var columnStart = "C";
|
||||
var columnEnd = GetColumnLetter(codesRow.Count + 1); // +1 because we start from C (index 2)
|
||||
var range = $"{googleSheetName}!{columnStart}20:{columnEnd}20";
|
||||
|
||||
var update = _googleSheetValues.Update(valueRange, sheetId, range);
|
||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
update.Execute();
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Updated yearly summary data in row 20 with {CodeCount} totals",
|
||||
ProcessorType, codesRow.Count);
|
||||
}
|
||||
|
||||
private void UpdateMonthlyDataFromCodes(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||
{
|
||||
const string googleSheetName = "Raport_R6_DRAFT_2025";
|
||||
|
||||
if (processedLayer.Records == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: No records found in processed layer", ProcessorType);
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a dictionary to store updates for each cell
|
||||
var cellUpdates = new Dictionary<string, string>();
|
||||
|
||||
foreach (var record in processedLayer.Records)
|
||||
{
|
||||
if (string.IsNullOrEmpty(record.Code) || record.Code.Length != 6)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Invalid code format in record {RecordId}: {Code}",
|
||||
ProcessorType, record.Id, record.Code);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Extract 4-digit code and month from 6-digit code
|
||||
var fourDigitCode = record.Code.Substring(0, 4);
|
||||
var monthStr = record.Code.Substring(4, 2);
|
||||
|
||||
if (!int.TryParse(monthStr, out var month) || month < 1 || month > 12)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Invalid month in code {Code}", ProcessorType, record.Code);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Find column index for the 4-digit code
|
||||
var columnIndex = -1;
|
||||
for (var i = 0; i < codesRow.Count; i++)
|
||||
{
|
||||
if (codesRow[i]?.ToString() == fourDigitCode)
|
||||
{
|
||||
columnIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (columnIndex == -1)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Code {FourDigitCode} not found in sheet header",
|
||||
ProcessorType, fourDigitCode);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate row (month 01 = row 7, month 02 = row 8, etc.)
|
||||
var targetRow = 6 + month; // row 7 for month 01, row 8 for month 02, etc.
|
||||
|
||||
// Calculate column letter (C = index 0, D = index 1, etc.)
|
||||
var targetColumn = GetColumnLetter(columnIndex + 2); // +2 because C is the first column (index 0)
|
||||
|
||||
var cellAddress = $"{targetColumn}{targetRow}";
|
||||
var value = record.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0";
|
||||
|
||||
cellUpdates[cellAddress] = value;
|
||||
|
||||
_logger.LogDebug(
|
||||
"{ProcessorType}: Mapping code {SixDigitCode} (base: {FourDigitCode}, month: {Month}) to cell {CellAddress} with value {Value}",
|
||||
ProcessorType, record.Code, fourDigitCode, month, cellAddress, value);
|
||||
}
|
||||
|
||||
// Batch update all cells
|
||||
if (cellUpdates.Any())
|
||||
{
|
||||
var batchUpdateRequest = new BatchUpdateValuesRequest
|
||||
{
|
||||
ValueInputOption = "USER_ENTERED",
|
||||
Data = cellUpdates.Select(kvp => new ValueRange
|
||||
{
|
||||
Range = $"{googleSheetName}!{kvp.Key}",
|
||||
Values = new List<IList<object>> { new List<object> { kvp.Value } }
|
||||
}).ToList()
|
||||
};
|
||||
|
||||
var batchUpdate = _googleSheetValues.BatchUpdate(batchUpdateRequest, sheetId);
|
||||
batchUpdate.Execute();
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Updated {CellCount} cells in Google Sheet",
|
||||
ProcessorType, cellUpdates.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: No valid data found to update in Google Sheet", ProcessorType);
|
||||
}
|
||||
}
|
||||
|
||||
private string GetColumnLetter(int columnIndex)
|
||||
{
|
||||
var columnLetter = string.Empty;
|
||||
while (columnIndex >= 0)
|
||||
{
|
||||
columnLetter = (char)('A' + (columnIndex % 26)) + columnLetter;
|
||||
columnIndex = columnIndex / 26 - 1;
|
||||
}
|
||||
|
||||
return columnLetter;
|
||||
}
|
||||
|
||||
private void UpdateTimestamps(string sheetId, Layer processedLayer)
|
||||
{
|
||||
const string googleSheetName = "Raport_R6_DRAFT_2025";
|
||||
|
||||
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var valueRangeTime = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>
|
||||
{
|
||||
new List<object> { timeUtc },
|
||||
new List<object> { timeWarsaw }
|
||||
}
|
||||
};
|
||||
|
||||
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{googleSheetName}!G1:G2");
|
||||
updateTime.ValueInputOption =
|
||||
SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateTime.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||
ProcessorType, timeUtc, timeWarsaw);
|
||||
}
|
||||
}
|
||||
500
DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs
Normal file
500
DiunaBI.Plugins.Morska/Processors/MorskaT1R1Processor.cs
Normal file
@@ -0,0 +1,500 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using DiunaBI.Infrastructure.Services.Calculations;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT1R1Processor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T1.R1";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT1R1Processor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private List<Record>? Sources { get; set; }
|
||||
private List<Record>? DynamicCodes { get; set; }
|
||||
private string? GoogleSheetName { get; set; }
|
||||
|
||||
public MorskaT1R1Processor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT1R1Processor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load sources
|
||||
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||
if (Sources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Source records not found");
|
||||
}
|
||||
|
||||
// Load dynamic codes
|
||||
DynamicCodes = processWorker.Records
|
||||
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||
.ToList();
|
||||
|
||||
// Load Google Sheet name
|
||||
GoogleSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
|
||||
if (string.IsNullOrEmpty(GoogleSheetName))
|
||||
{
|
||||
throw new InvalidOperationException("GoogleSheetName record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}, SheetName: {SheetName}",
|
||||
ProcessorType, Year, Sources.Count, DynamicCodes.Count, GoogleSheetName);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||
if (string.IsNullOrEmpty(GoogleSheetName)) errors.Add("GoogleSheetName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
|
||||
ProcessorType, Year, Sources!.Count);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Process records for each month
|
||||
var newRecords = ProcessAllMonths();
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
// Update Google Sheet report
|
||||
UpdateGoogleSheetReport(processedLayer.Id);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R1-T1";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Record> ProcessAllMonths()
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
for (var month = 1; month < 14; month++)
|
||||
{
|
||||
// Skip future months (except month 13 which is summary)
|
||||
if (Year > DateTime.UtcNow.Year ||
|
||||
(Year == DateTime.UtcNow.Year && month > DateTime.UtcNow.Month && month != 13))
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Skipping future month {Year}/{Month:D2}",
|
||||
ProcessorType, Year, month);
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing month {Month} for year {Year}",
|
||||
ProcessorType, month, Year);
|
||||
|
||||
var monthRecords = ProcessSingleMonth(month);
|
||||
newRecords.AddRange(monthRecords);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records for month {Month}",
|
||||
ProcessorType, monthRecords.Count, month);
|
||||
}
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessSingleMonth(int month)
|
||||
{
|
||||
var records = new List<Record>();
|
||||
|
||||
// Collect records from all sources
|
||||
foreach (var source in Sources!)
|
||||
{
|
||||
var sourceRecords = GetSourceRecords(source, month);
|
||||
records.AddRange(sourceRecords);
|
||||
}
|
||||
|
||||
// Process dynamic codes (calculations)
|
||||
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||
{
|
||||
var calculatedRecords = ProcessDynamicCodes(records, month);
|
||||
records.AddRange(calculatedRecords);
|
||||
}
|
||||
|
||||
// Create final records with month suffix
|
||||
var monthRecords = records.Select(x => new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{x.Code}{month:D2}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = x.Value32
|
||||
}).ToList();
|
||||
|
||||
return monthRecords;
|
||||
}
|
||||
|
||||
private List<Record> GetSourceRecords(Record source, int month)
|
||||
{
|
||||
var dataSource = _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSource == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Source layer {Year}/{month:D2}-{source.Desc1}-T3 not found");
|
||||
}
|
||||
|
||||
var sourceRecords = new List<Record>();
|
||||
|
||||
// Check if there are specific codes configured for this source
|
||||
var codesRecord = Sources!
|
||||
.Where(x => x.Code == $"Codes-{source.Desc1}")
|
||||
.FirstOrDefault();
|
||||
|
||||
if (codesRecord != null)
|
||||
{
|
||||
var codes = ProcessHelper.ParseCodes(codesRecord.Desc1!);
|
||||
sourceRecords.AddRange(dataSource.Records!
|
||||
.Where(x => codes.Contains(int.Parse(x.Code!))));
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using filtered codes for source {Source}: {CodeCount} codes",
|
||||
ProcessorType, source.Desc1, codes.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
sourceRecords.AddRange(dataSource.Records!);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using all records for source {Source}: {RecordCount} records",
|
||||
ProcessorType, source.Desc1, dataSource.Records!.Count);
|
||||
}
|
||||
|
||||
return sourceRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessDynamicCodes(List<Record> baseRecords, int month)
|
||||
{
|
||||
var calculatedRecords = new List<Record>();
|
||||
|
||||
foreach (var dynamicCode in DynamicCodes!)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing for month {Month}",
|
||||
ProcessorType, dynamicCode.Id, month);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||
if (!calc.IsFormulaCorrect())
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct for month {Month}",
|
||||
ProcessorType, calc.Expression, dynamicCode.Id, month);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calculatedRecord = calc.CalculateT1(baseRecords.Concat(calculatedRecords).ToList());
|
||||
calculatedRecords.Add(calculatedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code} for month {Month}, result: {Value}",
|
||||
ProcessorType, calculatedRecord.Code, month, calculatedRecord.Value32);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error for month {Month}",
|
||||
ProcessorType, dynamicCode.Desc1, month);
|
||||
}
|
||||
}
|
||||
|
||||
return calculatedRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Updating Google Sheet report {SheetName}",
|
||||
ProcessorType, GoogleSheetName);
|
||||
|
||||
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";
|
||||
|
||||
// Get processed layer data
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.Id == sourceId)
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||
}
|
||||
|
||||
// Get codes from sheet header
|
||||
var codesResponse = _googleSheetValues.Get(sheetId, $"{GoogleSheetName}!C4:DS4").Execute();
|
||||
var codesRow = codesResponse.Values[0];
|
||||
|
||||
// Update monthly data (months 1-12)
|
||||
UpdateMonthlyData(sheetId, codesRow, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
|
||||
// Update summary row (month 13)
|
||||
UpdateSummaryData(sheetId, codesRow, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
// Update timestamps
|
||||
UpdateTimestamps(sheetId, processedLayer);
|
||||
Thread.Sleep(1000);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet report {SheetName}",
|
||||
ProcessorType, GoogleSheetName);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report {SheetName}",
|
||||
ProcessorType, GoogleSheetName);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateMonthlyData(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||
{
|
||||
var valueRange = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>()
|
||||
};
|
||||
|
||||
// Process months 1-12
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var monthValues = new List<object>();
|
||||
foreach (string code in codesRow)
|
||||
{
|
||||
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{month:D2}");
|
||||
monthValues.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0");
|
||||
}
|
||||
valueRange.Values.Add(monthValues);
|
||||
}
|
||||
|
||||
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C7:DS18");
|
||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
update.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated monthly data in Google Sheet", ProcessorType);
|
||||
}
|
||||
|
||||
private void UpdateSummaryData(string sheetId, IList<object> codesRow, Layer processedLayer)
|
||||
{
|
||||
var valueRange = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>()
|
||||
};
|
||||
|
||||
// Empty row
|
||||
var emptyRow = new List<object>();
|
||||
for (int i = 0; i < codesRow.Count; i++)
|
||||
{
|
||||
emptyRow.Add("");
|
||||
}
|
||||
valueRange.Values.Add(emptyRow);
|
||||
|
||||
// Summary row (month 13)
|
||||
var summaryValues = new List<object>();
|
||||
foreach (string code in codesRow)
|
||||
{
|
||||
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
||||
summaryValues.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0");
|
||||
}
|
||||
valueRange.Values.Add(summaryValues);
|
||||
|
||||
var update = _googleSheetValues.Update(valueRange, sheetId, $"{GoogleSheetName}!C19:DS20");
|
||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
update.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated summary data in Google Sheet", ProcessorType);
|
||||
}
|
||||
|
||||
private void UpdateTimestamps(string sheetId, Layer processedLayer)
|
||||
{
|
||||
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var valueRangeTime = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>
|
||||
{
|
||||
new List<object> { timeUtc },
|
||||
new List<object> { timeWarsaw }
|
||||
}
|
||||
};
|
||||
|
||||
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{GoogleSheetName}!G1:G2");
|
||||
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateTime.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||
ProcessorType, timeUtc, timeWarsaw);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
475
DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs
Normal file
475
DiunaBI.Plugins.Morska/Processors/MorskaT1R3Processor.cs
Normal file
@@ -0,0 +1,475 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT1R3Processor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T1.R3";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT1R3Processor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private string? Source { get; set; }
|
||||
|
||||
public MorskaT1R3Processor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT1R3Processor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load source
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
|
||||
ProcessorType, Year, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Source: {Source}",
|
||||
ProcessorType, Year, Source);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
// Update Google Sheet report
|
||||
UpdateGoogleSheetReport(processedLayer.Id);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}-R3-T1";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
string pattern = @$"^L\d+-P-{Year}/\d+-{Source}-T5$";
|
||||
|
||||
var dataSources = _db.Layers
|
||||
.Where(x => !x.IsDeleted && !x.IsCancelled)
|
||||
.Include(layer => layer.Records!)
|
||||
.AsNoTracking()
|
||||
.AsEnumerable()
|
||||
.Where(x => Regex.IsMatch(x.Name!, pattern))
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for pattern: {pattern}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources matching pattern {Pattern}",
|
||||
ProcessorType, dataSources.Count, pattern);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
foreach (var dataSource in dataSources)
|
||||
{
|
||||
var monthStr = ProcessHelper.ExtractMonthFromLayerName(dataSource.Name!);
|
||||
if (monthStr == null || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Could not extract month from layer name: {LayerName}",
|
||||
ProcessorType, dataSource.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing data source {LayerName} for month {Month}",
|
||||
ProcessorType, dataSource.Name, month);
|
||||
|
||||
var sourceRecords = ProcessDataSourceRecords(dataSource, month);
|
||||
newRecords.AddRange(sourceRecords);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed {RecordCount} records from source {LayerName}",
|
||||
ProcessorType, sourceRecords.Count, dataSource.Name);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Total processed records: {TotalRecordCount}",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessDataSourceRecords(Layer dataSource, int month)
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
// L8542-D-DEPARTMENTS
|
||||
var dictionary = _db.Layers.Include(x => x.Records).FirstOrDefault(x => x.Number == 8542);
|
||||
|
||||
foreach (var record in dataSource.Records!)
|
||||
{
|
||||
if (record.Value1 == null)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Skipping record {RecordCode} - Value1 is null",
|
||||
ProcessorType, record.Code);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Process values for positions 1-32
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var value = ProcessHelper.GetValue(record, i);
|
||||
if (value == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var baseValue = (double)record.Value1!;
|
||||
var positionValue = (double)value;
|
||||
var calculatedValue = i == 1 ? baseValue : baseValue * positionValue / 100;
|
||||
|
||||
var newRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{record.Code}{month:D2}{i:D2}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = calculatedValue,
|
||||
Desc1 = record.Desc1
|
||||
};
|
||||
|
||||
newRecords.Add(newRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created record {NewRecordCode} with value {Value} from {OriginalCode}",
|
||||
ProcessorType, newRecord.Code, newRecord.Value1, record.Code);
|
||||
}
|
||||
}
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Starting Google Sheet report update for layer {LayerId}",
|
||||
ProcessorType, sourceId);
|
||||
|
||||
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";
|
||||
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.Id == sourceId)
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||
}
|
||||
|
||||
// Update sheets for all months
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
UpdateMonthSheet(sheetId, processedLayer, month);
|
||||
Thread.Sleep(1000);
|
||||
}
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheet reports for all months",
|
||||
ProcessorType);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheet report for layer {LayerId}",
|
||||
ProcessorType, sourceId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateMonthSheet(string sheetId, Layer processedLayer, int month)
|
||||
{
|
||||
var sheetName = ProcessHelper.GetSheetName(month, Year);
|
||||
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Updating sheet {SheetName} for month {Month}",
|
||||
ProcessorType, sheetName, month);
|
||||
|
||||
// Get codes from sheet
|
||||
ValueRange? dataRangeResponse;
|
||||
try
|
||||
{
|
||||
dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetName}!A7:A200").Execute();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Sheet {SheetName} not accessible, skipping - {Error}",
|
||||
ProcessorType, sheetName, e.Message);
|
||||
return;
|
||||
}
|
||||
|
||||
if (dataRangeResponse?.Values == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: No data found in sheet {SheetName}, skipping",
|
||||
ProcessorType, sheetName);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update data
|
||||
UpdateSheetData(sheetId, sheetName, processedLayer, dataRangeResponse.Values, month);
|
||||
|
||||
// Update timestamps
|
||||
UpdateSheetTimestamps(sheetId, sheetName, processedLayer);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully updated sheet {SheetName}",
|
||||
ProcessorType, sheetName);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to update sheet {SheetName} for month {Month}",
|
||||
ProcessorType, sheetName, month);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateSheetData(string sheetId, string sheetName, Layer processedLayer, IList<IList<object>> codeRows, int month)
|
||||
{
|
||||
var updateValueRange = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>()
|
||||
};
|
||||
|
||||
foreach (var row in codeRows)
|
||||
{
|
||||
if (row.Count == 0) continue;
|
||||
|
||||
var code = row[0].ToString();
|
||||
var updateRow = new List<object>();
|
||||
|
||||
var department = "";
|
||||
// Process columns C to Q (positions 1-15)
|
||||
for (var position = 1; position <= 15; position++)
|
||||
{
|
||||
var recordCode = $"{code}{month:D2}{position:D2}";
|
||||
var codeRecord = processedLayer.Records!.FirstOrDefault(x => x.Code == recordCode);
|
||||
|
||||
if (codeRecord?.Value1 != null)
|
||||
{
|
||||
updateRow.Add(codeRecord.Value1);
|
||||
_logger.LogDebug("{ProcessorType}: Found value {Value} for code {RecordCode}",
|
||||
ProcessorType, codeRecord.Value1, recordCode);
|
||||
}
|
||||
else
|
||||
{
|
||||
updateRow.Add("");
|
||||
}
|
||||
department = codeRecord?.Desc1 ?? "";
|
||||
}
|
||||
|
||||
updateRow.Add(department);
|
||||
|
||||
updateValueRange.Values.Add(updateRow);
|
||||
}
|
||||
|
||||
// Update sheet with new values
|
||||
var update = _googleSheetValues.Update(updateValueRange, sheetId, $"{sheetName}!C7:R200");
|
||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
update.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated {RowCount} rows of data in sheet {SheetName}",
|
||||
ProcessorType, updateValueRange.Values.Count, sheetName);
|
||||
}
|
||||
|
||||
private void UpdateSheetTimestamps(string sheetId, string sheetName, Layer processedLayer)
|
||||
{
|
||||
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var valueRangeTime = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>>
|
||||
{
|
||||
new List<object> { timeUtc },
|
||||
new List<object> { timeWarsaw }
|
||||
}
|
||||
};
|
||||
|
||||
var updateTime = _googleSheetValues.Update(valueRangeTime, sheetId, $"{sheetName}!G1:G2");
|
||||
updateTime.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateTime.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated timestamps in sheet {SheetName} - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||
ProcessorType, sheetName, timeUtc, timeWarsaw);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "T3.MultiSourceCopySelectedCodes";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT3MultiSourceCopySelectedCodesProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private List<Record>? Sources { get; set; }
|
||||
private string? Codes { get; set; }
|
||||
private List<int>? CodesList { get; set; }
|
||||
|
||||
public MorskaT3MultiSourceCopySelectedCodesProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT3MultiSourceCopySelectedCodesProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load month
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load sources
|
||||
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||
if (Sources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Source records not found");
|
||||
}
|
||||
|
||||
// Load codes
|
||||
Codes = GetRecordValue(processWorker.Records, "Codes");
|
||||
if (string.IsNullOrEmpty(Codes))
|
||||
{
|
||||
throw new InvalidOperationException("Codes record not found");
|
||||
}
|
||||
|
||||
// Parse codes list
|
||||
CodesList = ProcessHelper.ParseCodes(Codes);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, Codes: {CodeCount}",
|
||||
ProcessorType, Year, Month, Sources.Count, CodesList.Count);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||
if (CodesList == null || CodesList.Count == 0) errors.Add("No codes configured");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
|
||||
ProcessorType, Year, Month, Sources!.Count);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AB-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = new List<Layer>();
|
||||
|
||||
foreach (var source in Sources!)
|
||||
{
|
||||
var dataSource = _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSource == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Data source not found for {Year}/{Month:D2}-{Source}-T3",
|
||||
ProcessorType, Year, Month, source.Desc1);
|
||||
continue;
|
||||
}
|
||||
|
||||
dataSources.Add(dataSource);
|
||||
_logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
|
||||
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||
}
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources",
|
||||
ProcessorType, dataSources.Count);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allSourceRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
|
||||
var filteredRecords = allSourceRecords
|
||||
.Where(x => !string.IsNullOrEmpty(x.Code) &&
|
||||
int.TryParse(x.Code, out var code) &&
|
||||
CodesList!.Contains(code))
|
||||
.ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Filtered {FilteredCount} records from {TotalCount} total records using {CodeCount} codes",
|
||||
ProcessorType, filteredRecords.Count, allSourceRecords.Count, CodesList!.Count);
|
||||
|
||||
var newRecords = filteredRecords.Select(x => CreateCopiedRecord(x)).ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} copied records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private Record CreateCopiedRecord(Record sourceRecord)
|
||||
{
|
||||
var newRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = sourceRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Copy all values from positions 1-32
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var value = ProcessHelper.GetValue(sourceRecord, i);
|
||||
ProcessHelper.SetValue(newRecord, i, value);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Copied record {Code} with values from positions 1-32",
|
||||
ProcessorType, newRecord.Code);
|
||||
|
||||
return newRecord;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
|
||||
public MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}",
|
||||
ProcessorType, Year);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}",
|
||||
ProcessorType, Year);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources for all months
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records (sum all monthly values)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AB-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = new List<Layer>();
|
||||
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var dataSource = _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-AB-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSource != null)
|
||||
{
|
||||
dataSources.Add(dataSource);
|
||||
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
|
||||
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
|
||||
ProcessorType, month);
|
||||
}
|
||||
}
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for year {Year}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
|
||||
ProcessorType, dataSources.Count, Year);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var baseRecords = dataSources.Last().Records!;
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
|
||||
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
|
||||
|
||||
foreach (var baseRecord in baseRecords)
|
||||
{
|
||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = baseRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Sum values from all months for positions 1-32
|
||||
for (var position = 1; position <= 32; position++)
|
||||
{
|
||||
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position) ?? 0);
|
||||
ProcessHelper.SetValue(processedRecord, position, totalValue);
|
||||
}
|
||||
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
|
||||
ProcessorType, baseRecord.Code, codeRecords.Count);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using DiunaBI.Infrastructure.Services.Calculations;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3MultiSourceSummaryProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T3.MultiSourceSummary";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT3MultiSourceSummaryProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private List<Record>? Sources { get; set; }
|
||||
private List<Record>? DynamicCodes { get; set; }
|
||||
|
||||
public MorskaT3MultiSourceSummaryProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT3MultiSourceSummaryProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load month
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load sources
|
||||
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||
if (Sources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Source records not found");
|
||||
}
|
||||
|
||||
// Load dynamic codes
|
||||
DynamicCodes = processWorker.Records
|
||||
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||
.ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
|
||||
ProcessorType, Year, Month, Sources.Count, DynamicCodes.Count);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month} with {SourceCount} sources",
|
||||
ProcessorType, Year, Month, Sources!.Count);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records (sum by base codes)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Process dynamic codes if configured
|
||||
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||
{
|
||||
var calculatedRecords = ProcessDynamicCodes(newRecords);
|
||||
newRecords.AddRange(calculatedRecords);
|
||||
}
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-AA-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = Sources!.Select(source =>
|
||||
_db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{Month:D2}-{source.Desc1}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault())
|
||||
.OfType<Layer>()
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for {Year}/{Month:D2}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
|
||||
ProcessorType, dataSources.Count);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
|
||||
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
|
||||
|
||||
foreach (var baseCode in baseCodes)
|
||||
{
|
||||
var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
|
||||
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"9{baseCode}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Sum values from all sources for positions 1-32
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||
ProcessHelper.SetValue(processedRecord, i, totalValue);
|
||||
}
|
||||
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
|
||||
ProcessorType, baseCode, codeRecords.Count);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
|
||||
{
|
||||
var calculatedRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
|
||||
ProcessorType, DynamicCodes!.Count);
|
||||
|
||||
foreach (var dynamicCode in DynamicCodes!)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
|
||||
ProcessorType, dynamicCode.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||
if (!calc.IsFormulaCorrect())
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
|
||||
ProcessorType, calc.Expression, dynamicCode.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calculatedRecord = calc.CalculateT3(baseRecords.Concat(calculatedRecords).ToList());
|
||||
calculatedRecords.Add(calculatedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
|
||||
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
|
||||
ProcessorType, dynamicCode.Desc1);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
|
||||
ProcessorType, calculatedRecords.Count);
|
||||
|
||||
return calculatedRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using DiunaBI.Infrastructure.Services.Calculations;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T3.MultiSourceYearSummary";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT3MultiSourceYearSummaryProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private List<Record>? Sources { get; set; }
|
||||
private List<Record>? DynamicCodes { get; set; }
|
||||
|
||||
public MorskaT3MultiSourceYearSummaryProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT3MultiSourceYearSummaryProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load sources
|
||||
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||
if (Sources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Source records not found");
|
||||
}
|
||||
|
||||
// Load dynamic codes
|
||||
DynamicCodes = processWorker.Records
|
||||
.Where(x => x.Code!.Contains("DynamicCode-"))
|
||||
.OrderBy(x => int.Parse(x.Code!.Split('-')[1]))
|
||||
.ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, DynamicCodes: {DynamicCodeCount}",
|
||||
ProcessorType, Year, Sources.Count, DynamicCodes.Count);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year} with {SourceCount} sources",
|
||||
ProcessorType, Year, Sources!.Count);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records (sum by base codes with validation)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Process dynamic codes if configured
|
||||
if (DynamicCodes != null && DynamicCodes.Count > 0)
|
||||
{
|
||||
var calculatedRecords = ProcessDynamicCodes(newRecords);
|
||||
newRecords.AddRange(calculatedRecords);
|
||||
}
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-AA-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = Sources!.Select(source =>
|
||||
_db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault())
|
||||
.OfType<Layer>()
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for year {Year}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}",
|
||||
ProcessorType, dataSources.Count, Year);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} records from {DataSourceCount} sources, found {BaseCodeCount} base codes",
|
||||
ProcessorType, allRecords.Count, dataSources.Count, baseCodes.Count);
|
||||
|
||||
foreach (var baseCode in baseCodes)
|
||||
{
|
||||
var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode).ToList();
|
||||
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"9{baseCode}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Validation record for double-checking calculations
|
||||
var validationRecord = new Record();
|
||||
|
||||
// Sum values from all sources for positions 1-32 with validation
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||
ProcessHelper.SetValue(processedRecord, i, totalValue);
|
||||
|
||||
// Validation calculation (identical to main calculation)
|
||||
var validationValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, i));
|
||||
ProcessHelper.SetValue(validationRecord, i, validationValue);
|
||||
|
||||
// Validate that both calculations match
|
||||
var difference = Math.Abs((double)(ProcessHelper.GetValue(processedRecord, i) - ProcessHelper.GetValue(validationRecord, i))!);
|
||||
if (difference > 0.01)
|
||||
{
|
||||
throw new InvalidOperationException($"ValidationError: Code {baseCode}, Value{i} ({ProcessHelper.GetValue(processedRecord, i)} | {ProcessHelper.GetValue(validationRecord, i)})");
|
||||
}
|
||||
}
|
||||
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed base code {BaseCode} - summed values from {RecordCount} source records",
|
||||
ProcessorType, baseCode, codeRecords.Count);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessDynamicCodes(List<Record> baseRecords)
|
||||
{
|
||||
var calculatedRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {DynamicCodeCount} dynamic codes",
|
||||
ProcessorType, DynamicCodes!.Count);
|
||||
|
||||
foreach (var dynamicCode in DynamicCodes!)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrEmpty(dynamicCode.Desc1))
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula in Record {RecordId} is missing",
|
||||
ProcessorType, dynamicCode.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calc = new BaseCalc(dynamicCode.Desc1);
|
||||
if (!calc.IsFormulaCorrect())
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Formula {Expression} in Record {RecordId} is not correct",
|
||||
ProcessorType, calc.Expression, dynamicCode.Id);
|
||||
continue;
|
||||
}
|
||||
|
||||
var calculatedRecord = calc.CalculateT3(baseRecords.Concat(calculatedRecords).ToList());
|
||||
calculatedRecords.Add(calculatedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully calculated dynamic code {Code}, result: {Value}",
|
||||
ProcessorType, calculatedRecord.Code, calculatedRecord.Value1);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogWarning(e, "{ProcessorType}: Formula {Expression} calculation error",
|
||||
ProcessorType, dynamicCode.Desc1);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Successfully calculated {CalculatedCount} dynamic records",
|
||||
ProcessorType, calculatedRecords.Count);
|
||||
|
||||
return calculatedRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,340 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Google.Apis.Sheets.v4;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3SingleSourceProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T3.SingleSource";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT3SingleSourceProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private string? SourceLayerName { get; set; }
|
||||
private string? Source { get; set; }
|
||||
private Layer? SourceImportWorker { get; set; }
|
||||
|
||||
public MorskaT3SingleSourceProcessor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT3SingleSourceProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year and month
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load source layer name
|
||||
SourceLayerName = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||
if (string.IsNullOrEmpty(SourceLayerName))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayer record not found");
|
||||
}
|
||||
|
||||
// Load source name
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||
ProcessorType, Year, Month, SourceLayerName, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (string.IsNullOrEmpty(SourceLayerName)) errors.Add("SourceLayer is required");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
// Find source import worker
|
||||
SourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == SourceLayerName && !x.IsDeleted && !x.IsCancelled);
|
||||
if (SourceImportWorker == null)
|
||||
{
|
||||
errors.Add($"SourceImportWorker layer '{SourceLayerName}' not found");
|
||||
}
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||
ProcessorType, Year, Month, Source);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == SourceImportWorker!.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderBy(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for import worker '{SourceImportWorker!.Name}'");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for processing",
|
||||
ProcessorType, dataSources.Count);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing records from {RecordCount} total records",
|
||||
ProcessorType, allRecords.Count);
|
||||
|
||||
foreach (var baseRecord in dataSources.Last().Records!)
|
||||
{
|
||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = baseRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
ProcessDailyValues(processedRecord, codeRecords);
|
||||
|
||||
newRecords.Add(processedRecord);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed {ProcessedRecordCount} records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private void ProcessDailyValues(Record processedRecord, List<Record> codeRecords)
|
||||
{
|
||||
var lastDayInMonth = DateTime.DaysInMonth(Year, Month);
|
||||
|
||||
// Day 1 - first value for the month
|
||||
var firstVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Date <= new DateTime(Year, Month, 1))
|
||||
.MaxBy(x => x.CreatedAt)?.Value1 ?? 0;
|
||||
|
||||
ProcessHelper.SetValue(processedRecord, 1, firstVal);
|
||||
var previousValue = firstVal;
|
||||
|
||||
// Days 2 to last-1 - daily differences
|
||||
for (var i = 2; i < lastDayInMonth; i++)
|
||||
{
|
||||
var dayVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Day == i && x.CreatedAt.Month == Month)
|
||||
.MaxBy(x => x.CreatedAt)?.Value1;
|
||||
|
||||
if (dayVal == null)
|
||||
{
|
||||
ProcessHelper.SetValue(processedRecord, i, 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
var processedVal = dayVal - previousValue;
|
||||
ProcessHelper.SetValue(processedRecord, i, processedVal);
|
||||
previousValue = (double)dayVal;
|
||||
}
|
||||
}
|
||||
|
||||
// Last day - special handling
|
||||
var lastVal = codeRecords
|
||||
.Where(x => x.CreatedAt.Date >= new DateTime(Year, Month, lastDayInMonth))
|
||||
.MaxBy(x => x.CreatedAt)?.Value1;
|
||||
|
||||
if (lastVal == null)
|
||||
{
|
||||
ProcessHelper.SetValue(processedRecord, lastDayInMonth, 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
ProcessHelper.SetValue(processedRecord, lastDayInMonth, (double)lastVal - previousValue);
|
||||
}
|
||||
|
||||
// Copy last value to position 32
|
||||
var valueToCopy = codeRecords.MaxBy(x => x.CreatedAt)?.Value1;
|
||||
ProcessHelper.SetValue(processedRecord, 32, valueToCopy);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed daily values for code {Code}, last value: {LastValue}",
|
||||
ProcessorType, processedRecord.Code, valueToCopy);
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var isNew = processedLayer.Id == Guid.Empty || !_db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (isNew)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Google.Apis.Sheets.v4;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT3SourceYearSummaryProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T3.SourceYearSummary";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT3SourceYearSummaryProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private string? Source { get; set; }
|
||||
|
||||
public MorskaT3SourceYearSummaryProcessor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT3SourceYearSummaryProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load source
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Source: {Source}",
|
||||
ProcessorType, Year, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing year summary for Year: {Year}, Source: {Source}",
|
||||
ProcessorType, Year, Source);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources for all months
|
||||
var dataSources = GetDataSources();
|
||||
|
||||
// Process records (sum all monthly values)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/13-{Source}-T3";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources()
|
||||
{
|
||||
var dataSources = new List<Layer>();
|
||||
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var dataSource = _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{Source}-T3"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSource != null)
|
||||
{
|
||||
dataSources.Add(dataSource);
|
||||
_logger.LogDebug("{ProcessorType}: Found data source for month {Month}: {LayerName} with {RecordCount} records",
|
||||
ProcessorType, month, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: No data source found for month {Month}",
|
||||
ProcessorType, month);
|
||||
}
|
||||
}
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No data sources found for year {Year}, source {Source}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for year {Year}, source {Source}",
|
||||
ProcessorType, dataSources.Count, Year, Source);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var baseRecords = dataSources.Last().Records!;
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {AllRecordCount} total records from {MonthCount} months, using {BaseRecordCount} base records",
|
||||
ProcessorType, allRecords.Count, dataSources.Count, baseRecords.Count);
|
||||
|
||||
foreach (var baseRecord in baseRecords)
|
||||
{
|
||||
var codeRecords = allRecords.Where(x => x.Code == baseRecord.Code).ToList();
|
||||
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = baseRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Sum values from all months for positions 1-32
|
||||
for (var position = 1; position <= 32; position++)
|
||||
{
|
||||
var totalValue = codeRecords.Sum(x => ProcessHelper.GetValue(x, position));
|
||||
ProcessHelper.SetValue(processedRecord, position, totalValue);
|
||||
}
|
||||
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed code {Code} - summed values from {RecordCount} monthly records",
|
||||
ProcessorType, baseRecord.Code, codeRecords.Count);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} summary records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
600
DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs
Normal file
600
DiunaBI.Plugins.Morska/Processors/MorskaT4R2Processor.cs
Normal file
@@ -0,0 +1,600 @@
|
||||
using System.Globalization;
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Google.Apis.Sheets.v4;
|
||||
using Google.Apis.Sheets.v4.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT4R2Processor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T4.R2";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT4R2Processor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private List<Record>? Sources { get; set; }
|
||||
private string? LayerName { get; set; }
|
||||
private string? ReportSheetName { get; set; }
|
||||
private string? InvoicesSheetName { get; set; }
|
||||
|
||||
public MorskaT4R2Processor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT4R2Processor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load sources
|
||||
Sources = processWorker.Records.Where(x => x.Code == "Source").ToList();
|
||||
if (Sources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Source records not found");
|
||||
}
|
||||
|
||||
// Load layer name
|
||||
LayerName = GetRecordValue(processWorker.Records, "LayerName");
|
||||
if (string.IsNullOrEmpty(LayerName))
|
||||
{
|
||||
throw new InvalidOperationException("LayerName record not found");
|
||||
}
|
||||
|
||||
// Load report sheet name
|
||||
ReportSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName");
|
||||
if (string.IsNullOrEmpty(ReportSheetName))
|
||||
{
|
||||
throw new InvalidOperationException("GoogleSheetName record not found");
|
||||
}
|
||||
|
||||
// Load invoices sheet name
|
||||
InvoicesSheetName = GetRecordValue(processWorker.Records, "GoogleSheetName-Invoices");
|
||||
if (string.IsNullOrEmpty(InvoicesSheetName))
|
||||
{
|
||||
throw new InvalidOperationException("GoogleSheetName-Invoices record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Sources: {SourceCount}, LayerName: {LayerName}",
|
||||
ProcessorType, Year, Sources.Count, LayerName);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Sources == null || Sources.Count == 0) errors.Add("No sources configured");
|
||||
if (string.IsNullOrEmpty(LayerName)) errors.Add("LayerName is required");
|
||||
if (string.IsNullOrEmpty(ReportSheetName)) errors.Add("ReportSheetName is required");
|
||||
if (string.IsNullOrEmpty(InvoicesSheetName)) errors.Add("InvoicesSheetName is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year} with {SourceCount} sources",
|
||||
ProcessorType, Year, Sources!.Count);
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Process records for all sources
|
||||
var newRecords = ProcessSources();
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
// Update Google Sheets reports
|
||||
UpdateGoogleSheetReport(processedLayer.Id);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-{LayerName}";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Record> ProcessSources()
|
||||
{
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
foreach (var source in Sources!)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing source {Source}",
|
||||
ProcessorType, source.Desc1);
|
||||
|
||||
var sourceCodes = GetSourceCodes(source);
|
||||
var sourceRecords = ProcessSourceData(source, sourceCodes);
|
||||
newRecords.AddRange(sourceRecords);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed source {Source} - created {RecordCount} records",
|
||||
ProcessorType, source.Desc1, sourceRecords.Count);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Total records created: {TotalRecordCount}",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private List<int> GetSourceCodes(Record source)
|
||||
{
|
||||
var rawSourceCodes = GetRecordValue(Sources!, $"Codes-{source.Desc1}");
|
||||
if (string.IsNullOrEmpty(rawSourceCodes))
|
||||
{
|
||||
return new List<int>();
|
||||
}
|
||||
|
||||
return ProcessHelper.ParseCodes(rawSourceCodes);
|
||||
}
|
||||
|
||||
private List<Record> ProcessSourceData(Record source, List<int> sourceCodes)
|
||||
{
|
||||
var sourceRecords = new List<Record>();
|
||||
var lastSourceCodes = new List<string>();
|
||||
|
||||
// Process monthly data (1-12)
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var monthRecords = ProcessMonthData(source, sourceCodes, month, lastSourceCodes);
|
||||
sourceRecords.AddRange(monthRecords);
|
||||
}
|
||||
|
||||
// Process year summary (month 13)
|
||||
var yearSummaryRecords = ProcessYearSummaryData(source, sourceCodes);
|
||||
sourceRecords.AddRange(yearSummaryRecords);
|
||||
|
||||
return sourceRecords;
|
||||
}
|
||||
|
||||
private List<Record> ProcessMonthData(Record source, List<int> sourceCodes, int month, List<string> lastSourceCodes)
|
||||
{
|
||||
var monthRecords = new List<Record>();
|
||||
|
||||
if (IsDataAvailableForMonth(month))
|
||||
{
|
||||
var dataSource = GetMonthDataSource(source, month);
|
||||
if (dataSource != null)
|
||||
{
|
||||
lastSourceCodes.Clear();
|
||||
lastSourceCodes.AddRange(dataSource.Records!.Select(x => x.Code!));
|
||||
|
||||
var filteredRecords = FilterRecords(dataSource.Records!, sourceCodes);
|
||||
var processedRecords = CreateMonthRecords(filteredRecords, source, month);
|
||||
monthRecords.AddRange(processedRecords);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed month {Month} for source {Source} - {RecordCount} records",
|
||||
ProcessorType, month, source.Desc1, processedRecords.Count);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Data source {DataSource} not found",
|
||||
ProcessorType, $"{Year}/{month:D2}-{source.Desc1}-T");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Future months - create zero value records (except for FK2)
|
||||
if (source.Desc1 != "FK2" && lastSourceCodes.Count > 0)
|
||||
{
|
||||
var futureRecords = CreateFutureMonthRecords(lastSourceCodes, sourceCodes, month);
|
||||
monthRecords.AddRange(futureRecords);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {RecordCount} zero-value records for future month {Month}",
|
||||
ProcessorType, futureRecords.Count, month);
|
||||
}
|
||||
}
|
||||
|
||||
return monthRecords;
|
||||
}
|
||||
|
||||
private bool IsDataAvailableForMonth(int month)
|
||||
{
|
||||
return (Year == DateTime.UtcNow.Year && month <= DateTime.UtcNow.Month) || Year < DateTime.UtcNow.Year;
|
||||
}
|
||||
|
||||
private Layer? GetMonthDataSource(Record source, int month)
|
||||
{
|
||||
return _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/{month:D2}-{source.Desc1}-T"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
private List<Record> FilterRecords(IEnumerable<Record> records, List<int> sourceCodes)
|
||||
{
|
||||
return records
|
||||
.Where(x => sourceCodes.Count <= 0 || sourceCodes.Contains(int.Parse(x.Code!)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private List<Record> CreateMonthRecords(List<Record> filteredRecords, Record source, int month)
|
||||
{
|
||||
return filteredRecords.Select(x => new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{x.Code}{month:D2}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = source.Desc1 != "FK2" ? x.Value32 : x.Value1,
|
||||
Desc1 = x.Desc1
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
private List<Record> CreateFutureMonthRecords(List<string> lastSourceCodes, List<int> sourceCodes, int month)
|
||||
{
|
||||
return lastSourceCodes
|
||||
.Where(x => sourceCodes.Contains(int.Parse(x)))
|
||||
.Select(x => new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{x}{month:D2}",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = 0
|
||||
}).ToList();
|
||||
}
|
||||
|
||||
private List<Record> ProcessYearSummaryData(Record source, List<int> sourceCodes)
|
||||
{
|
||||
var dataSourceSum = _db.Layers
|
||||
.Where(x => x.Type == LayerType.Processed &&
|
||||
!x.IsDeleted && !x.IsCancelled &&
|
||||
x.Name != null && x.Name.Contains($"{Year}/13-{source.Desc1}-T"))
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSourceSum == null)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Year summary data source {DataSource} not found",
|
||||
ProcessorType, $"{Year}/13-{source.Desc1}-T3");
|
||||
return new List<Record>();
|
||||
}
|
||||
|
||||
var filteredRecords = FilterRecords(dataSourceSum.Records!, sourceCodes);
|
||||
var yearSummaryRecords = filteredRecords.Select(x => new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = $"{x.Code}13",
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow,
|
||||
Value1 = x.Value32
|
||||
}).ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {RecordCount} year summary records for source {Source}",
|
||||
ProcessorType, yearSummaryRecords.Count, source.Desc1);
|
||||
|
||||
return yearSummaryRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private void UpdateGoogleSheetReport(Guid sourceId)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Starting Google Sheets report update for layer {LayerId}",
|
||||
ProcessorType, sourceId);
|
||||
|
||||
const string sheetId = "1FsUmk_YRIeeGzFCX9tuUJCaLyRtjutX2ZGAEU1DMfJQ";
|
||||
|
||||
var processedLayer = GetProcessedLayerData(sourceId);
|
||||
if (processedLayer == null)
|
||||
{
|
||||
throw new InvalidOperationException($"Processed layer {sourceId} not found");
|
||||
}
|
||||
|
||||
var codesRow = GetCodesFromSheet(sheetId);
|
||||
|
||||
UpdateMonthlyData(sheetId, processedLayer, codesRow);
|
||||
UpdateYearSummary(sheetId, processedLayer, codesRow);
|
||||
UpdateTimestamps(sheetId, processedLayer);
|
||||
UpdateInvoicesData(sheetId, processedLayer);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully updated Google Sheets reports",
|
||||
ProcessorType);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to update Google Sheets report for layer {LayerId}",
|
||||
ProcessorType, sourceId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private Layer? GetProcessedLayerData(Guid sourceId)
|
||||
{
|
||||
return _db.Layers
|
||||
.Where(x => x.Id == sourceId && !x.IsDeleted && !x.IsCancelled)
|
||||
.Include(x => x.Records)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
private IList<object> GetCodesFromSheet(string sheetId)
|
||||
{
|
||||
var request = _googleSheetValues.Get(sheetId, "C4:Z4");
|
||||
var response = request.Execute();
|
||||
return response.Values[0];
|
||||
}
|
||||
|
||||
private void UpdateMonthlyData(string sheetId, Layer processedLayer, IList<object> codesRow)
|
||||
{
|
||||
const int startRow = 6;
|
||||
|
||||
for (var month = 1; month <= 12; month++)
|
||||
{
|
||||
var values = new List<object>();
|
||||
var monthStr = month < 10 ? $"0{month}" : month.ToString();
|
||||
|
||||
foreach (string code in codesRow)
|
||||
{
|
||||
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}{monthStr}");
|
||||
values.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0");
|
||||
}
|
||||
|
||||
var valueRange = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { values }
|
||||
};
|
||||
|
||||
var row = (startRow + month).ToString();
|
||||
var update = _googleSheetValues.Update(valueRange, sheetId, $"{ReportSheetName}!C{row}:XZ{row}");
|
||||
update.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
update.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated month {Month} data in Google Sheet",
|
||||
ProcessorType, month);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateYearSummary(string sheetId, Layer processedLayer, IList<object> codesRow)
|
||||
{
|
||||
const int startRow = 6;
|
||||
var valuesSum = new List<object>();
|
||||
var emptyRow = new List<object>();
|
||||
|
||||
foreach (string code in codesRow)
|
||||
{
|
||||
var record = processedLayer.Records?.SingleOrDefault(x => x.Code == $"{code}13");
|
||||
emptyRow.Add("");
|
||||
valuesSum.Add(record?.Value1?.ToString(CultureInfo.GetCultureInfo("pl-PL")) ?? "0");
|
||||
}
|
||||
|
||||
// Insert empty row before sum
|
||||
var rowEmpty = (startRow + 13).ToString();
|
||||
var valueRangeEmpty = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { emptyRow }
|
||||
};
|
||||
var updateEmpty = _googleSheetValues.Update(valueRangeEmpty, sheetId, $"{ReportSheetName}!C{rowEmpty}:XZ{rowEmpty}");
|
||||
updateEmpty.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateEmpty.Execute();
|
||||
|
||||
// Update sum row
|
||||
var rowSum = (startRow + 14).ToString();
|
||||
var valueRangeSum = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { valuesSum }
|
||||
};
|
||||
var updateSum = _googleSheetValues.Update(valueRangeSum, sheetId, $"{ReportSheetName}!C{rowSum}:XZ{rowSum}");
|
||||
updateSum.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateSum.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated year summary data in Google Sheet", ProcessorType);
|
||||
}
|
||||
|
||||
private void UpdateTimestamps(string sheetId, Layer processedLayer)
|
||||
{
|
||||
// Update UTC time
|
||||
var timeUtc = processedLayer.ModifiedAt.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
var valueRangeUtcTime = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { new List<object> { timeUtc } }
|
||||
};
|
||||
var updateTimeUtc = _googleSheetValues.Update(valueRangeUtcTime, sheetId, $"{ReportSheetName}!G1");
|
||||
updateTimeUtc.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateTimeUtc.Execute();
|
||||
|
||||
// Update Warsaw time
|
||||
var warsawTimeZone = TimeZoneInfo.FindSystemTimeZoneById("Central European Standard Time");
|
||||
var warsawTime = TimeZoneInfo.ConvertTimeFromUtc(processedLayer.ModifiedAt.ToUniversalTime(), warsawTimeZone);
|
||||
var timeWarsaw = warsawTime.ToString("dd.MM.yyyy HH:mm:ss", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
var valueRangeWarsawTime = new ValueRange
|
||||
{
|
||||
Values = new List<IList<object>> { new List<object> { timeWarsaw } }
|
||||
};
|
||||
var updateTimeWarsaw = _googleSheetValues.Update(valueRangeWarsawTime, sheetId, $"{ReportSheetName}!G2");
|
||||
updateTimeWarsaw.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateTimeWarsaw.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated timestamps in Google Sheet - UTC: {TimeUtc}, Warsaw: {TimeWarsaw}",
|
||||
ProcessorType, timeUtc, timeWarsaw);
|
||||
}
|
||||
|
||||
private void UpdateInvoicesData(string sheetId, Layer processedLayer)
|
||||
{
|
||||
var invoices = processedLayer.Records!
|
||||
.Where(x => x.Code!.Length == 12)
|
||||
.OrderByDescending(x => x.Code)
|
||||
.ToList();
|
||||
|
||||
var invoicesValues = new List<IList<object>>();
|
||||
var cleanUpValues = new List<IList<object>>();
|
||||
|
||||
foreach (var invoice in invoices)
|
||||
{
|
||||
var invoiceDate = DateTime.ParseExact(invoice.Code!.Substring(0, 8), "yyyyMMdd", CultureInfo.InvariantCulture)
|
||||
.ToString("dd.MM.yyyy", CultureInfo.GetCultureInfo("pl-PL"));
|
||||
|
||||
var invoiceRow = new List<object>
|
||||
{
|
||||
invoiceDate, "", invoice.Desc1!, invoice.Value1!
|
||||
};
|
||||
invoicesValues.Add(invoiceRow);
|
||||
|
||||
var cleanupRow = new List<object> { "", "", "", "" };
|
||||
cleanUpValues.Add(cleanupRow);
|
||||
}
|
||||
|
||||
// Clear existing data
|
||||
var cleanupValueRange = new ValueRange { Values = cleanUpValues };
|
||||
var cleanupInvoices = _googleSheetValues.Update(cleanupValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
|
||||
cleanupInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
cleanupInvoices.Execute();
|
||||
|
||||
// Update with new data
|
||||
var invoicesValueRange = new ValueRange { Values = invoicesValues };
|
||||
var updateInvoices = _googleSheetValues.Update(invoicesValueRange, sheetId, $"{InvoicesSheetName}!A6:E");
|
||||
updateInvoices.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.USERENTERED;
|
||||
updateInvoices.Execute();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Updated {InvoiceCount} invoices in Google Sheet",
|
||||
ProcessorType, invoices.Count);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Google.Apis.Sheets.v4;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT4SingleSourceProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T4.SingleSource";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
|
||||
private readonly ILogger<MorskaT4SingleSourceProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private string? SourceLayer { get; set; }
|
||||
private string? Source { get; set; }
|
||||
|
||||
public MorskaT4SingleSourceProcessor(
|
||||
AppDbContext db,
|
||||
SpreadsheetsResource.ValuesResource googleSheetValues,
|
||||
ILogger<MorskaT4SingleSourceProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_googleSheetValues = googleSheetValues;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load month
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load source layer
|
||||
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||
if (string.IsNullOrEmpty(SourceLayer))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayer record not found");
|
||||
}
|
||||
|
||||
// Load source
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||
ProcessorType, Year, Month, SourceLayer, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||
ProcessorType, Year, Month, Source);
|
||||
|
||||
// Get source import worker
|
||||
var sourceImportWorker = GetSourceImportWorker();
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data source
|
||||
var dataSource = GetDataSource(sourceImportWorker);
|
||||
|
||||
// Process records (simple copy)
|
||||
var newRecords = ProcessRecords(dataSource);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetSourceImportWorker()
|
||||
{
|
||||
var sourceImportWorker = _db.Layers
|
||||
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (sourceImportWorker == null)
|
||||
{
|
||||
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
|
||||
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
|
||||
|
||||
return sourceImportWorker;
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T4";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private Layer GetDataSource(Layer sourceImportWorker)
|
||||
{
|
||||
var dataSource = _db.Layers
|
||||
.Where(x => x.ParentId == sourceImportWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.Include(x => x.Records)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.FirstOrDefault();
|
||||
|
||||
if (dataSource == null)
|
||||
{
|
||||
throw new InvalidOperationException($"DataSource not found for source import worker: {sourceImportWorker.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found data source {LayerName} with {RecordCount} records",
|
||||
ProcessorType, dataSource.Name, dataSource.Records?.Count ?? 0);
|
||||
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(Layer dataSource)
|
||||
{
|
||||
if (dataSource.Records == null || dataSource.Records.Count == 0)
|
||||
{
|
||||
_logger.LogWarning("{ProcessorType}: Data source contains no records", ProcessorType);
|
||||
return new List<Record>();
|
||||
}
|
||||
|
||||
var newRecords = dataSource.Records.Select(record => new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = record.Code,
|
||||
Desc1 = record.Desc1,
|
||||
Value1 = record.Value1,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
}).ToList();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {RecordCount} copied records from data source",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
322
DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs
Normal file
322
DiunaBI.Plugins.Morska/Processors/MorskaT5LastValuesProcessor.cs
Normal file
@@ -0,0 +1,322 @@
|
||||
using DiunaBI.Domain.Entities;
|
||||
using DiunaBI.Infrastructure.Data;
|
||||
using DiunaBI.Infrastructure.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace DiunaBI.Plugins.Morska.Processors;
|
||||
|
||||
public class MorskaT5LastValuesProcessor : MorskaBaseProcessor
|
||||
{
|
||||
public override string ProcessorType => "Morska.Process.T5.LastValues";
|
||||
|
||||
private readonly AppDbContext _db;
|
||||
private readonly ILogger<MorskaT5LastValuesProcessor> _logger;
|
||||
|
||||
// Configuration properties loaded from layer records
|
||||
private int Year { get; set; }
|
||||
private int Month { get; set; }
|
||||
private string? SourceLayer { get; set; }
|
||||
private string? Source { get; set; }
|
||||
|
||||
public MorskaT5LastValuesProcessor(
|
||||
AppDbContext db,
|
||||
ILogger<MorskaT5LastValuesProcessor> logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public override void Process(Layer processWorker)
|
||||
{
|
||||
try
|
||||
{
|
||||
_logger.LogInformation("{ProcessorType}: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
|
||||
// Load configuration from layer records
|
||||
LoadConfiguration(processWorker);
|
||||
|
||||
// Validate required configuration
|
||||
ValidateConfiguration();
|
||||
|
||||
// Perform the actual processing
|
||||
PerformProcessing(processWorker);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully completed processing for {ProcessWorkerName}",
|
||||
ProcessorType, processWorker.Name);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "{ProcessorType}: Failed to process {ProcessWorkerName} ({ProcessWorkerId})",
|
||||
ProcessorType, processWorker.Name, processWorker.Id);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadConfiguration(Layer processWorker)
|
||||
{
|
||||
if (processWorker.Records == null)
|
||||
{
|
||||
throw new InvalidOperationException("ProcessWorker has no records");
|
||||
}
|
||||
|
||||
// Load year
|
||||
var yearStr = GetRecordValue(processWorker.Records, "Year");
|
||||
if (string.IsNullOrEmpty(yearStr) || !int.TryParse(yearStr, out var year))
|
||||
{
|
||||
throw new InvalidOperationException("Year record not found or invalid");
|
||||
}
|
||||
Year = year;
|
||||
|
||||
// Load month
|
||||
var monthStr = GetRecordValue(processWorker.Records, "Month");
|
||||
if (string.IsNullOrEmpty(monthStr) || !int.TryParse(monthStr, out var month))
|
||||
{
|
||||
throw new InvalidOperationException("Month record not found or invalid");
|
||||
}
|
||||
Month = month;
|
||||
|
||||
// Load source layer
|
||||
SourceLayer = GetRecordValue(processWorker.Records, "SourceLayer");
|
||||
if (string.IsNullOrEmpty(SourceLayer))
|
||||
{
|
||||
throw new InvalidOperationException("SourceLayer record not found");
|
||||
}
|
||||
|
||||
// Load source
|
||||
Source = GetRecordValue(processWorker.Records, "Source");
|
||||
if (string.IsNullOrEmpty(Source))
|
||||
{
|
||||
throw new InvalidOperationException("Source record not found");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration loaded - Year: {Year}, Month: {Month}, SourceLayer: {SourceLayer}, Source: {Source}",
|
||||
ProcessorType, Year, Month, SourceLayer, Source);
|
||||
}
|
||||
|
||||
private void ValidateConfiguration()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (Year < 2000 || Year > 3000) errors.Add($"Invalid year: {Year}");
|
||||
if (Month < 1 || Month > 12) errors.Add($"Invalid month: {Month}");
|
||||
if (string.IsNullOrEmpty(SourceLayer)) errors.Add("SourceLayer is required");
|
||||
if (string.IsNullOrEmpty(Source)) errors.Add("Source is required");
|
||||
|
||||
if (errors.Any())
|
||||
{
|
||||
throw new InvalidOperationException($"Configuration validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Configuration validation passed", ProcessorType);
|
||||
}
|
||||
|
||||
private void PerformProcessing(Layer processWorker)
|
||||
{
|
||||
_logger.LogDebug("{ProcessorType}: Processing data for Year: {Year}, Month: {Month}, Source: {Source}",
|
||||
ProcessorType, Year, Month, Source);
|
||||
|
||||
// Get source import worker
|
||||
var sourceImportWorker = GetSourceImportWorker();
|
||||
|
||||
// Get or create processed layer
|
||||
var processedLayer = GetOrCreateProcessedLayer(processWorker);
|
||||
|
||||
// Get data sources
|
||||
var dataSources = GetDataSources(sourceImportWorker);
|
||||
|
||||
// Process records (get last values for each code)
|
||||
var newRecords = ProcessRecords(dataSources);
|
||||
|
||||
// Save results
|
||||
SaveProcessedLayer(processedLayer, newRecords);
|
||||
|
||||
_logger.LogInformation("{ProcessorType}: Successfully processed {RecordCount} records for layer {LayerName} ({LayerId})",
|
||||
ProcessorType, newRecords.Count, processedLayer.Name, processedLayer.Id);
|
||||
}
|
||||
|
||||
private Layer GetSourceImportWorker()
|
||||
{
|
||||
var sourceImportWorker = _db.Layers
|
||||
.Where(x => x.Name == SourceLayer && !x.IsDeleted && !x.IsCancelled)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (sourceImportWorker == null)
|
||||
{
|
||||
throw new InvalidOperationException($"SourceImportWorker layer not found: {SourceLayer}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found source import worker {LayerName} ({LayerId})",
|
||||
ProcessorType, sourceImportWorker.Name, sourceImportWorker.Id);
|
||||
|
||||
return sourceImportWorker;
|
||||
}
|
||||
|
||||
private Layer GetOrCreateProcessedLayer(Layer processWorker)
|
||||
{
|
||||
var processedLayer = _db.Layers
|
||||
.Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (processedLayer == null)
|
||||
{
|
||||
processedLayer = new Layer
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Type = LayerType.Processed,
|
||||
ParentId = processWorker.Id,
|
||||
Number = _db.Layers.Count() + 1,
|
||||
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
processedLayer.Name = $"L{processedLayer.Number}-P-{Year}/{Month:D2}-{Source}-T5";
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created new processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
processedLayer.ModifiedAt = DateTime.UtcNow;
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Using existing processed layer {LayerName}",
|
||||
ProcessorType, processedLayer.Name);
|
||||
}
|
||||
|
||||
return processedLayer;
|
||||
}
|
||||
|
||||
private List<Layer> GetDataSources(Layer sourceImportWorker)
|
||||
{
|
||||
var dataSources = _db.Layers
|
||||
.Include(x => x.Records)
|
||||
.Where(x => x.ParentId == sourceImportWorker.Id &&
|
||||
!x.IsDeleted && !x.IsCancelled)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.AsNoTracking()
|
||||
.ToList();
|
||||
|
||||
if (dataSources.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"DataSource is empty for {sourceImportWorker.Name}");
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Found {DataSourceCount} data sources for {SourceWorkerName}",
|
||||
ProcessorType, dataSources.Count, sourceImportWorker.Name);
|
||||
|
||||
return dataSources;
|
||||
}
|
||||
|
||||
private List<Record> ProcessRecords(List<Layer> dataSources)
|
||||
{
|
||||
var allRecords = dataSources.SelectMany(x => x.Records!).ToList();
|
||||
var codes = allRecords.Select(x => x.Code).Distinct().ToList();
|
||||
var newRecords = new List<Record>();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processing {CodeCount} unique codes from {TotalRecordCount} total records",
|
||||
ProcessorType, codes.Count, allRecords.Count);
|
||||
|
||||
foreach (var code in codes)
|
||||
{
|
||||
var lastRecord = allRecords
|
||||
.Where(x => x.Code == code)
|
||||
.OrderByDescending(x => x.CreatedAt)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (lastRecord == null) continue;
|
||||
|
||||
var processedRecord = CreateProcessedRecord(lastRecord);
|
||||
newRecords.Add(processedRecord);
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Processed code {Code} - using record from {CreatedAt}",
|
||||
ProcessorType, code, lastRecord.CreatedAt);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Created {NewRecordCount} processed records",
|
||||
ProcessorType, newRecords.Count);
|
||||
|
||||
return newRecords;
|
||||
}
|
||||
|
||||
private Record CreateProcessedRecord(Record lastRecord)
|
||||
{
|
||||
var processedRecord = new Record
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
Code = lastRecord.Code,
|
||||
CreatedAt = DateTime.UtcNow,
|
||||
ModifiedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Copy all values from positions 1-32
|
||||
for (var i = 1; i < 33; i++)
|
||||
{
|
||||
var value = ProcessHelper.GetValue(lastRecord, i);
|
||||
if (value != null)
|
||||
{
|
||||
ProcessHelper.SetValue(processedRecord, i, value);
|
||||
}
|
||||
}
|
||||
|
||||
processedRecord.Desc1 = lastRecord.Desc1;
|
||||
|
||||
return processedRecord;
|
||||
}
|
||||
|
||||
private void SaveProcessedLayer(Layer processedLayer, List<Record> newRecords)
|
||||
{
|
||||
var existsInDb = _db.Layers.Any(x => x.Id == processedLayer.Id);
|
||||
|
||||
if (!existsInDb)
|
||||
{
|
||||
_db.Layers.Add(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Added new processed layer to database", ProcessorType);
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Layers.Update(processedLayer);
|
||||
_logger.LogDebug("{ProcessorType}: Updated existing processed layer in database", ProcessorType);
|
||||
}
|
||||
|
||||
SaveRecords(processedLayer.Id, newRecords);
|
||||
_db.SaveChanges();
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Saved {RecordCount} records for layer {LayerId}",
|
||||
ProcessorType, newRecords.Count, processedLayer.Id);
|
||||
}
|
||||
|
||||
private void SaveRecords(Guid layerId, ICollection<Record> records)
|
||||
{
|
||||
// Remove existing records for this layer
|
||||
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
|
||||
if (toDelete.Count > 0)
|
||||
{
|
||||
_db.Records.RemoveRange(toDelete);
|
||||
_logger.LogDebug("{ProcessorType}: Removed {DeletedCount} existing records for layer {LayerId}",
|
||||
ProcessorType, toDelete.Count, layerId);
|
||||
}
|
||||
|
||||
// Add new records
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.CreatedAt = DateTime.UtcNow;
|
||||
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
|
||||
record.ModifiedAt = DateTime.UtcNow;
|
||||
record.LayerId = layerId;
|
||||
_db.Records.Add(record);
|
||||
}
|
||||
|
||||
_logger.LogDebug("{ProcessorType}: Added {RecordCount} new records for layer {LayerId}",
|
||||
ProcessorType, records.Count, layerId);
|
||||
}
|
||||
|
||||
private string? GetRecordValue(ICollection<Record> records, string code)
|
||||
{
|
||||
return records.FirstOrDefault(x => x.Code == code)?.Desc1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user