mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Logging refactor

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-02 18:53:25 +02:00
parent b800890320
commit ab3310a0c2
25 changed files with 682 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs
View File

@@ -1,5 +1,6 @@
using DiunaBI.Core.Interfaces;
using DiunaBI.Core.Models;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;

86
src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs
View File

@@ -6,6 +6,7 @@ using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
using DiunaBI.Core.Services.Calculations;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -15,12 +16,16 @@ public class T1R1Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R1Processor> _logger;
public T1R1Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T1R1Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -108,34 +113,16 @@ public class T1R1Processor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -145,33 +132,14 @@ public class T1R1Processor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message =
$"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
@@ -195,8 +163,7 @@ public class T1R1Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: Save records to the layer
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
@@ -208,6 +175,27 @@ public class T1R1Processor : MorskaBaseProcessor
UpdateReport(processedLayer.Id, sheetName);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, string sheetName)
{
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";

34
src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs
View File

@@ -6,6 +6,7 @@ using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -15,12 +16,16 @@ public class T1R3Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R3Processor> _logger;
public T1R3Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T1R3Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -108,13 +113,34 @@ public class T1R3Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
UpdateReport(processedLayer.Id, year);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, int year)
{
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";

32
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs
View File

@@ -2,6 +2,7 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -10,10 +11,14 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
public override string ProcessorType => "T3.MultiSourceCopySelectedCodes";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesProcessor> _logger;
public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -98,8 +103,29 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: Save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

30
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs
View File

@@ -2,18 +2,23 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -100,8 +105,27 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

97
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs
View File

@@ -1,5 +1,6 @@
using DiunaBI.Core.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services.Calculations;
@@ -9,15 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceSummary";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceSummaryProcessor> _logger;
T3MultiSourceSummaryProcessor(
AppDbContext db)
public T3MultiSourceSummaryProcessor(
AppDbContext db,
ILogger<T3MultiSourceSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
@@ -72,7 +81,6 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
foreach (var baseCode in baseCodes)
{
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
@@ -95,6 +103,7 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
var dynamicCodes = processWorker.Records?
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0)
{
foreach (var dynamicCode in dynamicCodes)
@@ -103,33 +112,16 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -139,37 +131,18 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
if (isNew)
{
_db.Layers.Add(processedLayer);
@@ -178,8 +151,32 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

106
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs
View File

@@ -1,4 +1,5 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -9,14 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceYearSummary";
private readonly AppDbContext _db;
T3MultiSourceYearSummaryProcessor(
AppDbContext db)
private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger;
public T3MultiSourceYearSummaryProcessor(
AppDbContext db,
ILogger<T3MultiSourceYearSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
@@ -59,23 +69,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0)
{
throw new Exception("DataSources are empty");
}
if (dataSources.Count == 0)
{
throw new Exception("DataSourcesValidation are empty");
}
var allRecords = dataSources
.SelectMany(x => x.Records!).ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (var baseCode in baseCodes)
{
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
@@ -110,12 +115,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
newRecords.Add(processedRecord);
}
// Dynamic Codes
var dynamicCodes = processWorker.Records?
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0)
{
foreach (var dynamicCode in dynamicCodes)
@@ -124,33 +128,16 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -160,35 +147,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
if (isNew)
{
_db.Layers.Add(processedLayer);
@@ -197,8 +167,32 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

35
src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs
View File

@@ -2,17 +2,27 @@
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
namespace DiunaBI.Plugins.Morska.Processors;
public class T3SingleSourceProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.SingleSource";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SingleSourceProcessor> _logger;
public T3SingleSourceProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T3SingleSourceProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -139,9 +149,28 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

45
src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs
View File

@@ -1,4 +1,6 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -8,20 +10,33 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.SourceYearSummary";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SourceYearSummaryProcessor> _logger;
public T3SourceYearSummaryProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T3SourceYearSummaryProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception("Source record not found");
}
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
@@ -101,8 +116,32 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

69
src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs
View File

@@ -2,6 +2,7 @@
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -15,15 +16,23 @@ public class T4R2Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4R2Processor> _logger;
public T4R2Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T4R2Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
@@ -37,7 +46,6 @@ public class T4R2Processor : MorskaBaseProcessor
throw new Exception("LayerName record not found");
}
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
@@ -116,17 +124,8 @@ public class T4R2Processor : MorskaBaseProcessor
}
else
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/{month:D2}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
$"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
}
else
@@ -182,17 +181,8 @@ public class T4R2Processor : MorskaBaseProcessor
}
else
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
$"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
}
@@ -204,8 +194,8 @@ public class T4R2Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
@@ -219,7 +209,32 @@ public class T4R2Processor : MorskaBaseProcessor
{
throw new Exception("GoogleSheetName-Invoices record not found");
}
UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName);
_logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName)

52
src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs
View File

@@ -1,6 +1,8 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -9,13 +11,24 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
public override string ProcessorType => "T4.SingleSource";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4SingleSourceProcessor> _logger;
public T4SingleSourceProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T4SingleSourceProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
@@ -23,13 +36,15 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
{
throw new Exception("SourceLayer record not found");
}
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
throw new Exception("SourceImportWorker layer not found");
}
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null)
if (source == null)
{
throw new Exception("Source record not found");
}
@@ -61,7 +76,6 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
var dataSource = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
@@ -83,8 +97,7 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
Value1 = record.Value1,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
})
.ToList();
}).ToList();
if (isNew)
{
@@ -94,9 +107,32 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

30
src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs
View File

@@ -2,6 +2,7 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -10,10 +11,14 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
public override string ProcessorType => "T5.LastValues";
private readonly AppDbContext _db;
private readonly ILogger<T5LastValuesProcessor> _logger;
public T5LastValuesProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T5LastValuesProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -95,8 +100,27 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
_db.Layers.Add(processedLayer);
else
_db.Layers.Update(processedLayer);
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}