mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Logging refactor

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-02 18:53:25 +02:00
parent b800890320
commit ab3310a0c2
25 changed files with 682 additions and 750 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/build.yml vendored
View File

@@ -80,7 +80,6 @@ jobs:
run: |
rm -f appsettings.Development.json
rm -f client_secrets.Development.json
rm -f diunabi-admin-firebase-Development.json
- name: Upload artifact
uses: actions/upload-artifact@v4

1
.github/workflows/buildScripts/replaceTokens.js vendored
View File

@@ -10,7 +10,6 @@ module.exports = async ({ github, context, core, jobId }) => {
files.push(`./${jobId}/webapi/appsettings.json`);
files.push(`./${jobId}/webapi/client_secrets.json`);
files.push(`./${jobId}/webapi/diunabi-admin-firebase.json`);
files.forEach(file => {
let data = require('fs').readFileSync(file, 'utf8');

2
.gitignore vendored
View File

@@ -70,6 +70,4 @@ Temp/
# Sensitive files
**/appsettings.Development.json
**/firebase-adminsdk-*.json
src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase-Development.json

34
src/Backend/DiunaBI.Core/Models/LogEntry.cs
View File

@@ -1,34 +0,0 @@
using System;
namespace DiunaBI.Core.Models;
public enum LogEntryType
{
Info,
Warning,
Error
}
public enum LogType
{
Import,
Backup,
Process,
PowerBi,
DataInbox,
Queue
}
public enum LogInstance
{
Morska
}
public class LogEntry
{
public LogType LogType { get; init; }
public LogEntryType Type { get; init; }
public string? Message { get; init; }
public string? Title { get; init; }
public DateTime CreatedAt { get; init; }
public Guid SessionId { get; set; }
public LogInstance Instance { get; set; }
}

6
src/Backend/DiunaBI.Core/Services/PluginManager.cs
View File

@@ -46,8 +46,10 @@ public class PluginManager
}
}
_logger.LogInformation("Loaded {ProcessorCount} processors and {ImporterCount} importers from {PluginCount} plugins",
_processorTypes.Count, _importerTypes.Count, _plugins.Count);
_logger.LogInformation("Loaded {ProcessorCount} processors and {ImporterCount} importers from {AssemblyCount} assemblies",
_processorTypes.Count,
_importerTypes.Count,
dllFiles.Length); // Zmień z _plugins.Count na assemblyFiles.Length
}
private void LoadPluginFromAssembly(string assemblyPath)

11
src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj
View File

@@ -6,15 +6,12 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3525" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3624" />
<PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
</ItemGroup>
</Project>

1
src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs
View File

@@ -1,5 +1,6 @@
using DiunaBI.Core.Interfaces;
using DiunaBI.Core.Models;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;

86
src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs
View File

@@ -6,6 +6,7 @@ using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
using DiunaBI.Core.Services.Calculations;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -15,12 +16,16 @@ public class T1R1Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R1Processor> _logger;
public T1R1Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T1R1Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -108,34 +113,16 @@ public class T1R1Processor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -145,33 +132,14 @@ public class T1R1Processor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message =
$"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
//TODO throw exception or log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
@@ -195,8 +163,7 @@ public class T1R1Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: Save records to the layer
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
@@ -208,6 +175,27 @@ public class T1R1Processor : MorskaBaseProcessor
UpdateReport(processedLayer.Id, sheetName);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, string sheetName)
{
const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA";

34
src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs
View File

@@ -6,6 +6,7 @@ using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -15,12 +16,16 @@ public class T1R3Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T1R3Processor> _logger;
public T1R3Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T1R3Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -108,13 +113,34 @@ public class T1R3Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
UpdateReport(processedLayer.Id, year);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, int year)
{
const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8";

32
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs
View File

@@ -2,6 +2,7 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -10,10 +11,14 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
public override string ProcessorType => "T3.MultiSourceCopySelectedCodes";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesProcessor> _logger;
public T3MultiSourceCopySelectedCodesProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -98,8 +103,29 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: Save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

30
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs
View File

@@ -2,18 +2,23 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> _logger;
public T3MultiSourceCopySelectedCodesYearSummaryProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T3MultiSourceCopySelectedCodesYearSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -100,8 +105,27 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

97
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs
View File

@@ -1,5 +1,6 @@
using DiunaBI.Core.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services.Calculations;
@@ -9,15 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceSummary";
private readonly AppDbContext _db;
private readonly ILogger<T3MultiSourceSummaryProcessor> _logger;
T3MultiSourceSummaryProcessor(
AppDbContext db)
public T3MultiSourceSummaryProcessor(
AppDbContext db,
ILogger<T3MultiSourceSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
@@ -72,7 +81,6 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
foreach (var baseCode in baseCodes)
{
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
@@ -95,6 +103,7 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
var dynamicCodes = processWorker.Records?
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0)
{
foreach (var dynamicCode in dynamicCodes)
@@ -103,33 +112,16 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -139,37 +131,18 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
if (isNew)
{
_db.Layers.Add(processedLayer);
@@ -178,8 +151,32 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

106
src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs
View File

@@ -1,4 +1,5 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -9,14 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.MultiSourceYearSummary";
private readonly AppDbContext _db;
T3MultiSourceYearSummaryProcessor(
AppDbContext db)
private readonly ILogger<T3MultiSourceYearSummaryProcessor> _logger;
public T3MultiSourceYearSummaryProcessor(
AppDbContext db,
ILogger<T3MultiSourceYearSummaryProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
@@ -59,23 +69,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
.FirstOrDefault())
.OfType<Layer>()
.ToList();
if (dataSources.Count == 0)
{
throw new Exception("DataSources are empty");
}
if (dataSources.Count == 0)
{
throw new Exception("DataSourcesValidation are empty");
}
var allRecords = dataSources
.SelectMany(x => x.Records!).ToList();
var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList();
foreach (var baseCode in baseCodes)
{
var codeRecords = allRecords.Where(x =>
x.Code![1..] == baseCode)
.ToList();
@@ -110,12 +115,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
newRecords.Add(processedRecord);
}
// Dynamic Codes
var dynamicCodes = processWorker.Records?
.Where(x => x.Code!.Contains("DynamicCode-"))
.OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList();
if (dynamicCodes != null && dynamicCodes.Count != 0)
{
foreach (var dynamicCode in dynamicCodes)
@@ -124,33 +128,16 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
if (dynamicCode.Desc1 == null)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula in Record {dynamicCode.Id} is missing.",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
var calc = new BaseCalc(dynamicCode.Desc1);
if (!calc.IsFormulaCorrect())
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
continue;
}
@@ -160,35 +147,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})",
calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
catch (Exception e)
{
//TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Calculation error {dynamicCode.Id}: {e.Message} ",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})",
dynamicCode.Id, processWorker.Name, processWorker.Id);
}
}
}
if (isNew)
{
_db.Layers.Add(processedLayer);
@@ -197,8 +167,32 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

35
src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs
View File

@@ -2,17 +2,27 @@
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
namespace DiunaBI.Plugins.Morska.Processors;
public class T3SingleSourceProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.SingleSource";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SingleSourceProcessor> _logger;
public T3SingleSourceProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T3SingleSourceProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -139,9 +149,28 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

45
src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs
View File

@@ -1,4 +1,6 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -8,20 +10,33 @@ namespace DiunaBI.Plugins.Morska.Processors;
public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
{
public override string ProcessorType => "T3.SourceYearSummary";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T3SourceYearSummaryProcessor> _logger;
public T3SourceYearSummaryProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T3SourceYearSummaryProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (source == null)
{
throw new Exception("Source record not found");
}
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
@@ -101,8 +116,32 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
//TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

69
src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs
View File

@@ -2,6 +2,7 @@
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using DiunaBI.Core.Services;
@@ -15,15 +16,23 @@ public class T4R2Processor : MorskaBaseProcessor
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4R2Processor> _logger;
public T4R2Processor(
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T4R2Processor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList();
if (sources!.Count == 0)
@@ -37,7 +46,6 @@ public class T4R2Processor : MorskaBaseProcessor
throw new Exception("LayerName record not found");
}
var processedLayer = _db.Layers
.Where(x => x.ParentId == processWorker.Id
&& !x.IsDeleted && !x.IsCancelled)
@@ -116,17 +124,8 @@ public class T4R2Processor : MorskaBaseProcessor
}
else
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/{month:D2}-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
$"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
}
else
@@ -182,17 +181,8 @@ public class T4R2Processor : MorskaBaseProcessor
}
else
{
// TODO: log warning
/*
logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Process,
Message = $"Data source {year}/13-{source.Desc1}-T3 not found",
CreatedAt = DateTime.UtcNow
});
*/
_logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})",
$"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id);
}
}
@@ -204,8 +194,8 @@ public class T4R2Processor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1;
@@ -219,7 +209,32 @@ public class T4R2Processor : MorskaBaseProcessor
{
throw new Exception("GoogleSheetName-Invoices record not found");
}
UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName);
_logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName)

52
src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs
View File

@@ -1,6 +1,8 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
using Google.Apis.Sheets.v4;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -9,13 +11,24 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
public override string ProcessorType => "T4.SingleSource";
private readonly AppDbContext _db;
private readonly SpreadsheetsResource.ValuesResource _googleSheetValues;
private readonly ILogger<T4SingleSourceProcessor> _logger;
public T4SingleSourceProcessor(
AppDbContext db)
AppDbContext db,
SpreadsheetsResource.ValuesResource googleSheetValues,
ILogger<T4SingleSourceProcessor> logger)
{
_db = db;
_googleSheetValues = googleSheetValues;
_logger = logger;
}
public override void Process(Layer processWorker)
{
_logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!);
var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!);
var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1;
@@ -23,13 +36,15 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
{
throw new Exception("SourceLayer record not found");
}
var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled);
if (sourceImportWorker == null)
{
throw new Exception("SourceImportWorkerL layer not found");
throw new Exception("SourceImportWorker layer not found");
}
var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1;
if (sourceLayer == null)
if (source == null)
{
throw new Exception("Source record not found");
}
@@ -61,7 +76,6 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
processedLayer.ModifiedAt = DateTime.UtcNow;
var dataSource = _db.Layers
.Include(x => x.Records)
.Where(x => x.ParentId == sourceImportWorker.Id
@@ -83,8 +97,7 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
Value1 = record.Value1,
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow
})
.ToList();
}).ToList();
if (isNew)
{
@@ -94,9 +107,32 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor
{
_db.Layers.Update(processedLayer);
}
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
_logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})",
processWorker.Name, processWorker.Id);
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

30
src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs
View File

@@ -2,6 +2,7 @@
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Database.Context;
using Microsoft.Extensions.Logging;
namespace DiunaBI.Plugins.Morska.Processors;
@@ -10,10 +11,14 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
public override string ProcessorType => "T5.LastValues";
private readonly AppDbContext _db;
private readonly ILogger<T5LastValuesProcessor> _logger;
public T5LastValuesProcessor(
AppDbContext db)
AppDbContext db,
ILogger<T5LastValuesProcessor> logger)
{
_db = db;
_logger = logger;
}
public override void Process(Layer processWorker)
{
@@ -95,8 +100,27 @@ public class T5LastValuesProcessor : MorskaBaseProcessor
_db.Layers.Add(processedLayer);
else
_db.Layers.Update(processedLayer);
// TODO: save records
//controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"));
SaveRecords(processedLayer.Id, newRecords);
_db.SaveChanges();
}
private void SaveRecords(Guid layerId, ICollection<Record> records)
{
var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D");
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = layerId;
_db.Records.Add(record);
}
_logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId);
}
}

86
src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs
View File

@@ -2,8 +2,8 @@ using System.Text;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using DiunaBI.Database.Context;
using Google.Cloud.Firestore;
using DiunaBI.Core.Models;
namespace DiunaBI.WebAPI.Controllers;
@@ -14,16 +14,16 @@ public class DataInboxController : Controller
{
private readonly AppDbContext _db;
private readonly IConfiguration _configuration;
private readonly LogsController _logsController;
private readonly ILogger<DataInboxController> _logger;
public DataInboxController(
AppDbContext db,
IConfiguration configuration,
FirestoreDb firestoreDb)
ILogger<DataInboxController> logger)
{
_db = db;
_configuration = configuration;
_logsController = new LogsController(firestoreDb);
_logger = logger;
}
[HttpPut]
@@ -33,41 +33,22 @@ public class DataInboxController : Controller
{
if (apiKey != _configuration["apiKey"])
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong apiKey ({dataInbox.Source})",
Type = LogEntryType.Warning,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("DataInbox: Unauthorized request - wrong apiKey for source {Source}", dataInbox.Source);
return Unauthorized();
}
try
{
if (
!Request.Headers.TryGetValue("Authorization", out var authHeader))
if (!Request.Headers.TryGetValue("Authorization", out var authHeader))
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - no authorization header ({dataInbox.Source})",
Type = LogEntryType.Warning,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("DataInbox: Unauthorized request - no authorization header for source {Source}", dataInbox.Source);
return Unauthorized();
}
var credentialsArr = authHeader.ToString().Split(" ");
if (credentialsArr.Length != 2)
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong auth header format ({dataInbox.Source})",
Type = LogEntryType.Warning,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("DataInbox: Unauthorized request - wrong auth header format for source {Source}", dataInbox.Source);
return Unauthorized();
}
@@ -76,60 +57,35 @@ public class DataInboxController : Controller
var password = authValue.Split(':')[1];
if (username != _configuration["morska-user"] || password != _configuration["morska-pass"])
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - bad credentials ({dataInbox.Source})",
Type = LogEntryType.Warning,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("DataInbox: Unauthorized request - bad credentials for source {Source}", dataInbox.Source);
return Unauthorized();
}
// check if datainbox.data is base64 encoded value
if (!string.IsNullOrEmpty(dataInbox.Data) && !IsBase64String(dataInbox.Data))
{
_logsController.AddEntry(new LogEntry
{
Title = $"Invalid data format - not base64 encoded ({dataInbox.Source})",
Type = LogEntryType.Warning,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("DataInbox: Invalid data format - not base64 encoded for source {Source}", dataInbox.Source);
return BadRequest("Invalid data format - not base64 encoded");
}
dataInbox.Id = Guid.NewGuid();
dataInbox.CreatedAt = DateTime.UtcNow;
_db.DataInbox.Add(dataInbox);
_db.SaveChanges();
_logsController.AddEntry(new LogEntry
{
Title = $"Insert success: {dataInbox.Source}, {dataInbox.Name}",
Type = LogEntryType.Info,
LogType = LogType.DataInbox,
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("DataInbox: Insert success for source {Source}, name {Name}", dataInbox.Source, dataInbox.Name);
if (dataInbox.Name == "morska.d3.importer")
{
// TODO: import dataInbox as Layer
_logger.LogDebug("DataInbox: Detected morska.d3.importer - processing will be handled by AutoImport");
// AutoImport będzie obsługiwać ten typ danych
}
return Ok();
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = $"Insert error: {dataInbox.Source}, {dataInbox.Name}",
Type = LogEntryType.Error,
LogType = LogType.DataInbox,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "DataInbox: Insert error for source {Source}, name {Name}", dataInbox.Source, dataInbox.Name);
return BadRequest(e.ToString());
}
}
@@ -137,7 +93,17 @@ public class DataInboxController : Controller
[HttpGet]
public IActionResult GetAll()
{
return Ok(_db.DataInbox.AsNoTracking().ToList());
try
{
var dataInbox = _db.DataInbox.AsNoTracking().ToList();
_logger.LogDebug("DataInbox: Retrieved {Count} records", dataInbox.Count);
return Ok(dataInbox);
}
catch (Exception e)
{
_logger.LogError(e, "DataInbox: Error retrieving records");
return BadRequest(e.ToString());
}
}
// helpers
@@ -150,9 +116,7 @@ public class DataInboxController : Controller
try
{
var base64Bytes = Convert.FromBase64String(data);
var utf8String = Encoding.UTF8.GetString(base64Bytes);
var reEncoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(utf8String));
return data.TrimEnd('=') == reEncoded.TrimEnd('=');
}

558
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -19,28 +19,24 @@ public class LayersController : Controller
private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues;
private readonly GoogleDriveHelper _googleDriveHelper;
private readonly IConfiguration _configuration;
private readonly LogsController _logsController;
private readonly PluginManager _pluginManager;
private readonly ILogger<LayersController> _logger;
public LayersController(
AppDbContext db,
GoogleSheetsHelper googleSheetsHelper,
SpreadsheetsResource.ValuesResource? googleSheetValues,
GoogleDriveHelper googleDriveHelper,
IConfiguration configuration,
FirestoreDb firestoreDb,
PluginManager pluginManager
PluginManager pluginManager,
ILogger<LayersController> logger
)
{
_db = db;
if (googleSheetsHelper.Service is not null)
{
_googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values;
}
_googleSheetValues = googleSheetValues;
_googleDriveHelper = googleDriveHelper;
_configuration = configuration;
_logsController = new LogsController(firestoreDb);
_pluginManager = pluginManager;
_logger = logger;
}
[HttpGet]
@@ -59,12 +55,18 @@ public class LayersController : Controller
response = response.Where(x => x.Type == type);
}
return Ok(response
var result = response
.OrderByDescending(x => x.Number)
.Skip(start).Take(limit).AsNoTracking().ToList());
.Skip(start).Take(limit).AsNoTracking().ToList();
_logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}",
result.Count, name, type);
return Ok(result);
}
catch (Exception e)
{
_logger.LogError(e, "GetAll: Error retrieving layers");
return BadRequest(e.ToString());
}
}
@@ -75,13 +77,17 @@ public class LayersController : Controller
{
try
{
return Ok(_db.Layers
var layer = _db.Layers
.Include(x => x.CreatedBy)
.Include(x => x.ModifiedBy)
.Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted));
.Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted);
_logger.LogDebug("Get: Retrieved layer {LayerId} {LayerName}", id, layer.Name);
return Ok(layer);
}
catch (Exception e)
{
_logger.LogError(e, "Get: Error retrieving layer {LayerId}", id);
return BadRequest(e.ToString());
}
}
@@ -92,41 +98,22 @@ public class LayersController : Controller
{
if (apiKey != _configuration["apiKey"])
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong apiKey ({number})",
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("PowerBI: Unauthorized request - wrong apiKey for layer {LayerNumber}", number);
return Unauthorized();
}
try
{
if (
!Request.Headers.TryGetValue("Authorization", out var authHeader))
if (!Request.Headers.TryGetValue("Authorization", out var authHeader))
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - no authorization header ({number})",
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("PowerBI: Unauthorized request - no authorization header for layer {LayerNumber}", number);
return Unauthorized();
}
var credentialsArr = authHeader.ToString().Split(" ");
if (credentialsArr.Length != 2)
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - wrong auth header format ({number})",
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("PowerBI: Unauthorized request - wrong auth header format for layer {LayerNumber}", number);
return Unauthorized();
}
@@ -135,37 +122,21 @@ public class LayersController : Controller
var password = authValue.Split(':')[1];
if (username != _configuration["powerBI-user"] || password != _configuration["powerBI-pass"])
{
_logsController.AddEntry(new LogEntry
{
Title = $"Unauthorized request - bad credentials ({number})",
Type = LogEntryType.Warning,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogWarning("PowerBI: Unauthorized request - bad credentials for layer {LayerNumber}", number);
return Unauthorized();
}
_logsController.AddEntry(new LogEntry
{
Title = $"Sending data for layer {number}",
Type = LogEntryType.Info,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("PowerBI: Sending data for layer {LayerNumber}", number);
return Ok(_db.Layers
var layer = _db.Layers
.Include(x => x.CreatedBy)
.Include(x => x.Records).AsNoTracking().First(x => x.Number == number && !x.IsDeleted));
.Include(x => x.Records).AsNoTracking().First(x => x.Number == number && !x.IsDeleted);
return Ok(layer);
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = e.ToString(),
Type = LogEntryType.Error,
LogType = LogType.PowerBi,
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "PowerBI: Error occurred while processing layer {LayerNumber}", number);
return BadRequest(e.ToString());
}
}
@@ -176,20 +147,22 @@ public class LayersController : Controller
{
if (apiKey != _configuration["apiKey"])
{
_logger.LogWarning("Configuration: Unauthorized request - wrong apiKey for layer {LayerNumber}", number);
return Unauthorized();
}
try
{
if (
!Request.Headers.TryGetValue("Authorization", out var authHeader))
if (!Request.Headers.TryGetValue("Authorization", out var authHeader))
{
_logger.LogWarning("Configuration: Unauthorized request - no authorization header for layer {LayerNumber}", number);
return Unauthorized();
}
var credentialsArr = authHeader.ToString().Split(" ");
if (credentialsArr.Length != 2)
{
_logger.LogWarning("Configuration: Unauthorized request - wrong auth header format for layer {LayerNumber}", number);
return Unauthorized();
}
@@ -198,6 +171,7 @@ public class LayersController : Controller
var password = authValue.Split(':')[1];
if (username != _configuration["morska-user"] || password != _configuration["morska-pass"])
{
_logger.LogWarning("Configuration: Unauthorized request - bad credentials for layer {LayerNumber}", number);
return Unauthorized();
}
@@ -208,19 +182,23 @@ public class LayersController : Controller
if (config is null)
{
_logger.LogWarning("Configuration: Layer {LayerNumber} not found", number);
return BadRequest();
}
var type = config.Records?.Where(x => x.Code == "Type").FirstOrDefault();
if (type is null || type.Desc1 != "ExternalConfiguration")
{
_logger.LogWarning("Configuration: Layer {LayerNumber} is not ExternalConfiguration type", number);
return BadRequest();
}
_logger.LogInformation("Configuration: Sending configuration for layer {LayerNumber}", number);
return Ok(config);
}
catch
catch (Exception e)
{
_logger.LogError(e, "Configuration: Error occurred while processing layer {LayerNumber}", number);
return BadRequest();
}
}
@@ -231,19 +209,33 @@ public class LayersController : Controller
{
if (_googleSheetValues is null)
{
_logger.LogError("Export: Google Sheets API not initialized");
throw new Exception("Google Sheets API not initialized");
}
var layer = _db.Layers
.Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted);
var export = _pluginManager.GetExporter("GoogleSheet");
if (export == null)
try
{
throw new Exception("GoogleSheet exporter not found");
var layer = _db.Layers
.Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted);
var export = _pluginManager.GetExporter("GoogleSheet");
if (export == null)
{
_logger.LogError("Export: GoogleSheet exporter not found for layer {LayerId}", id);
throw new Exception("GoogleSheet exporter not found");
}
_logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name);
export.Export(layer);
_logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id);
return Ok(true);
}
catch (Exception e)
{
_logger.LogError(e, "Export: Failed to export layer {LayerId} to GoogleSheet", id);
throw;
}
export.Export(layer);
return Ok(true);
}
[HttpGet]
@@ -253,6 +245,7 @@ public class LayersController : Controller
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
@@ -268,38 +261,23 @@ public class LayersController : Controller
if (importWorkerLayers.Count == 0)
{
_logsController.AddEntry(new LogEntry
{
Title = "No Layers to import.",
Type = LogEntryType.Info,
LogType = LogType.Queue,
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("AutoImportQueue: No layers to import");
return Ok();
}
_logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count);
foreach (var importWorker in importWorkerLayers)
{
try
{
/*
await _queue.AddJob(new QueueJob
{
LayerId = importWorker.Id,
Type = JobType.ImportWorker,
});
*/
// Queue job implementation would go here
_logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id);
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = $"Error while adding job into queue (import): {importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Error,
LogType = LogType.Queue,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
}
return Ok();
@@ -310,17 +288,14 @@ public class LayersController : Controller
[AllowAnonymous]
public IActionResult ProcessQueue(string apiKey)
{
/*
var allJobs = await _queue.GetJobs();
var importJobs = allJobs
.Where(x => x.Type == JobType.ImportWorker && x.Status == JobStatus.New);
foreach (var job in importJobs)
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
job.Attempts = job.Attempts + 1;
//await _queue.UpdateJob(job);
_logger.LogWarning("ProcessQueue: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
*/
_logger.LogInformation("ProcessQueue: Starting queue processing");
// Queue processing implementation would go here
return Ok();
}
@@ -331,11 +306,13 @@ public class LayersController : Controller
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("AutoImport: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
if (_googleSheetValues is null)
{
_logger.LogError("AutoImport: Google Sheets API not initialized");
throw new Exception("Google Sheets API not initialized");
}
@@ -349,24 +326,15 @@ public class LayersController : Controller
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
_logsController.AddEntry(new LogEntry
{
Title = $"Starting import: {nameFilter}, Admin layers count ({importWorkerLayers.Count})",
Type = LogEntryType.Info,
LogType = LogType.Import,
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers",
nameFilter, importWorkerLayers.Count);
try
{
if (importWorkerLayers.Count == 0)
{
_logsController.AddEntry(new LogEntry
{
Title = "No Layers to import.",
Type = LogEntryType.Info,
LogType = LogType.Import,
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("AutoImport: No layers to import");
return Ok();
}
@@ -374,10 +342,12 @@ public class LayersController : Controller
{
try
{
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ??
"Standard";
var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ??
"GoogleSheet";
var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard";
var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet";
_logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}",
importWorker.Name, type, source);
if (source == "DataInbox" && type == "Import-D3")
{
var d3Importer = _pluginManager.GetImporter("MorskaD3");
@@ -387,16 +357,11 @@ public class LayersController : Controller
}
d3Importer.Import(importWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
continue;
}
switch (type)
{
case "D1":
@@ -406,134 +371,88 @@ public class LayersController : Controller
throw new Exception("MorskaD1 importer not found");
}
d1importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
_logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
break;
case "FK2":
var fk2importer = _pluginManager.GetImporter("MorskaFK2");
if (fk2importer == null)
{
var fk2importer = _pluginManager.GetImporter("MorskaFK2");
if (fk2importer == null)
throw new Exception("MorskaFK2 importer not found");
}
fk2importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
break;
default:
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
if (startDate == null)
{
throw new Exception("StartDate record not found");
}
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
if (endDate == null)
{
throw new Exception("EndDate record not found");
}
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date)
{
var importer = _pluginManager.GetImporter("MorskaImporter");
if (importer == null)
{
throw new Exception("MorskaFK2 importer not found");
throw new Exception("MorskaImporter not found");
}
fk2importer.Import(importWorker);
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
break;
_logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
default:
else if (IsImportedLayerUpToDate(importWorker) == false)
{
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
if (startDate == null)
var importer = _pluginManager.GetImporter("MorskaImporter");
if (importer == null)
{
throw new Exception("StartDate record nod found");
throw new Exception("MorskaImporter not found");
}
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
if (endDate == null)
{
throw new Exception("EndDate record nod found");
}
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date &&
endDateParsed.Date >= DateTime.UtcNow.Date)
{
var importer = _pluginManager.GetImporter("MorskaImporter");
if (importer == null)
{
throw new Exception("MorskaImporter not found");
}
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
}
else if (IsImportedLayerUpToDate(importWorker) == false)
{
var importer = _pluginManager.GetImporter("MorskaImporter");
if (importer == null)
{
throw new Exception("MorskaImporter not found");
}
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "Success (reimported)",
CreatedAt = DateTime.UtcNow
});
}
else
{
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "importLayer records are up to date. Not processed.",
CreatedAt = DateTime.UtcNow
});
}
break;
_logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
else
{
_logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping",
importWorker.Name, importWorker.Id);
}
break;
}
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Error,
LogType = LogType.Import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
}
_logger.LogInformation("AutoImport: Completed processing {LayerCount} layers", importWorkerLayers.Count);
return Ok();
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.Error,
LogType = LogType.Import,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "AutoImport: Process error");
return BadRequest(e.ToString());
}
}
@@ -545,11 +464,13 @@ public class LayersController : Controller
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("AutoProcess: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
if (_googleSheetValues is null)
{
_logger.LogError("AutoProcess: Google Sheets API not initialized");
throw new Exception("Google Sheets API not initialized");
}
@@ -557,8 +478,8 @@ public class LayersController : Controller
[
"T3-SingleSource",
"T3-SourceYearSummary",
"T3-MultiSourceSummary", // AA
"T3-MultiSourceYearSummary", // AA/13
"T3-MultiSourceSummary",
"T3-MultiSourceYearSummary",
"T4-SingleSource",
"T5-LastValues",
"T1-R1",
@@ -566,6 +487,8 @@ public class LayersController : Controller
"T1-R3"
];
_logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length);
foreach (var type in processTypes)
{
try
@@ -581,38 +504,31 @@ public class LayersController : Controller
.AsNoTracking()
.ToList();
_logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers",
type, processWorkerLayers.Count);
foreach (var processWorker in processWorkerLayers)
{
try
{
ProcessLayer(processWorker);
_logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}",
processWorker.Name, processWorker.Id, type);
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Error,
LogType = LogType.Process,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}",
processWorker.Name, processWorker.Id, type);
}
}
}
catch (Exception e)
{
_logsController.AddEntry(new LogEntry
{
Title = "Process error",
Type = LogEntryType.Error,
LogType = LogType.Process,
Message = e.ToString(),
CreatedAt = DateTime.UtcNow
});
_logger.LogError(e, "AutoProcess: Error processing type {ProcessType}", type);
}
}
_logger.LogInformation("AutoProcess: Completed processing all process types");
return Ok();
}
@@ -626,7 +542,7 @@ public class LayersController : Controller
var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1;
if (year == null)
{
throw new Exception("Year record nod found");
throw new Exception("Year record not found");
}
var processType = processWorker.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1;
@@ -642,15 +558,6 @@ public class LayersController : Controller
throw new Exception("T3.SourceYearSummary processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T3-MultiSourceYearSummary":
@@ -661,15 +568,6 @@ public class LayersController : Controller
throw new Exception("T3.MultiSourceYearSummary processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T3-MultiSourceCopySelectedCodesYearSummary":
@@ -680,15 +578,6 @@ public class LayersController : Controller
throw new Exception("T3.MultiSourceCopySelectedCodesYearSummary processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T1-R1":
@@ -699,15 +588,6 @@ public class LayersController : Controller
throw new Exception("T1.R1 processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T4-R2":
@@ -718,15 +598,6 @@ public class LayersController : Controller
throw new Exception("T4.R2 processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T1-R3":
@@ -737,15 +608,6 @@ public class LayersController : Controller
throw new Exception("T1.R3 processor not found");
}
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
}
@@ -809,35 +671,8 @@ public class LayersController : Controller
break;
}
}
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
}
internal void SaveRecords(Guid id, ICollection<Record> records, Guid currentUserId)
{
var toDelete = _db.Records.Where(x => x.LayerId == id).ToList();
if (toDelete.Count > 0)
{
_db.Records.RemoveRange(toDelete);
}
foreach (var record in records)
{
record.CreatedById = currentUserId;
record.CreatedAt = DateTime.UtcNow;
record.ModifiedById = currentUserId;
record.ModifiedAt = DateTime.UtcNow;
record.LayerId = id;
_db.Records.Add(record);
}
}
private static void WriteToConsole(params string[] messages)
{
@@ -863,6 +698,7 @@ public class LayersController : Controller
if (newestLayer is null)
{
_logger.LogDebug("IsImportedLayerUpToDate: No child layers found for {LayerName}, treating as up to date", importWorker.Name);
return true; // importWorker is not active yet, no check needed
}
@@ -884,39 +720,51 @@ public class LayersController : Controller
throw new Exception($"DataRange not found, {importWorker.Name}");
}
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
var isUpToDate = true;
for (var i = 0; i < data[1].Count; i++)
try
{
if (data[0][i].ToString() == "") continue;
var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (record == null)
var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute();
var data = dataRangeResponse.Values;
var isUpToDate = true;
for (var i = 0; i < data[1].Count; i++)
{
WriteToConsole("Code not found in DiunaBI", data[0][i].ToString()!);
if (data[0][i].ToString() == "") continue;
var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString());
if (record == null)
{
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}",
data[0][i].ToString(), importWorker.Name);
isUpToDate = false;
continue;
}
if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) ||
double.Abs((double)(record.Value1 - value)!) < 0.01) continue;
isUpToDate = false;
continue;
}
if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"),
out var value) ||
double.Abs((double)(record.Value1 - value)!) < 0.01) continue;
isUpToDate = false;
}
foreach (var record in newestLayer.Records!)
{
if (data[0].Contains(record.Code))
foreach (var record in newestLayer.Records!)
{
continue;
if (data[0].Contains(record.Code))
{
continue;
}
_logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}",
record.Code, importWorker.Name);
isUpToDate = false;
}
WriteToConsole($"Code not found in GoogleSheet: {record.Code}");
isUpToDate = false;
}
_logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}",
importWorker.Name, isUpToDate ? "up to date" : "outdated");
return isUpToDate;
return isUpToDate;
}
catch (Exception e)
{
_logger.LogError(e, "IsImportedLayerUpToDate: Error checking if layer {LayerName} is up to date", importWorker.Name);
throw;
}
}
}

46
src/Backend/DiunaBI.WebAPI/Controllers/LogsController.cs
View File

@@ -1,46 +0,0 @@
using Microsoft.AspNetCore.Mvc;
using DiunaBI.Core.Models;
using Google.Cloud.Firestore;
namespace DiunaBI.WebAPI.Controllers;
public class LogsController : Controller
{
private readonly FirestoreDb _firestoreDb;
private readonly Guid _SessionId = Guid.NewGuid();
public LogsController(
FirestoreDb firestoreDb
)
{
_firestoreDb = firestoreDb;
}
public void AddEntry(LogEntry entry)
{
entry.SessionId = _SessionId;
entry.Instance = LogInstance.Morska;
if (entry.Type == LogEntryType.Info) { return; }
try
{
var collection = _firestoreDb.Collection("ApiLogs");
var document = collection.Document();
document.SetAsync(new
{
entry.Message,
entry.Title,
Type = Enum.GetName(typeof(LogEntryType), entry.Type),
LogType = Enum.GetName(typeof(LogType), entry.LogType),
Instance = Enum.GetName(typeof(LogInstance), entry.Instance),
entry.CreatedAt,
SessionId = entry.SessionId.ToString()
}).Wait();
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}

1
src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj
View File

@@ -6,7 +6,6 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FirebaseAdmin" Version="2.4.0" />
<PackageReference Include="Google.Cloud.Firestore" Version="3.4.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.0" />
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />

18
src/Backend/DiunaBI.WebAPI/Program.cs
View File

@@ -1,4 +1,3 @@
using FirebaseAdmin;
using Google.Apis.Auth.OAuth2;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -87,24 +86,10 @@ builder.Services.AddSingleton<SpreadsheetsResource.ValuesResource>(provider =>
return valuesResource;
});
var fileName = "diunabi-admin-firebase.json";
#if DEBUG
fileName = "diunabi-admin-firebase-Development.json";
#endif
var credentialPath = Path.Combine(Directory.GetCurrentDirectory(), fileName);
System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialPath);
FirebaseAdmin.FirebaseApp.Create(new AppOptions()
{
Credential = GoogleCredential.GetApplicationDefault()
});
builder.Services.AddSingleton(FirestoreDb.Create("diunabi-admin"));
builder.Services.AddSingleton<PluginManager>();
var app = builder.Build();
// ✅ DODAJ SERILOG REQUEST LOGGING
app.UseSerilogRequestLogging(options =>
{
options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms";
@@ -113,20 +98,17 @@ app.UseSerilogRequestLogging(options =>
diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value);
diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme);
// ✅ POPRAW NULLABLE WARNING
var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault();
if (!string.IsNullOrEmpty(userAgent))
{
diagnosticContext.Set("UserAgent", userAgent);
}
// Dodaj więcej użytecznych właściwości
diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown");
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
};
});
// Załaduj pluginy - z logowaniem
var pluginManager = app.Services.GetRequiredService<PluginManager>();
var executablePath = Assembly.GetExecutingAssembly().Location;
var executableDir = Path.GetDirectoryName(executablePath)!;

38
src/Backend/DiunaBI.WebAPI/bin/Debug/net8.0/appsettings.json
View File

@@ -7,6 +7,44 @@
"Microsoft.AspNetCore": "Warning"
}
},
"Serilog": {
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft.AspNetCore": "Warning",
"Microsoft.EntityFrameworkCore.Database.Command": "Warning",
"Microsoft.EntityFrameworkCore.Infrastructure": "Warning",
"System.Net.Http.HttpClient": "Warning",
"Google.Apis": "Warning",
"DiunaBI.Core.Services.PluginManager": "Information"
}
},
"WriteTo": [
{
"Name": "Console",
"Args": {
"outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"path": "/var/log/diunabi/app-.log",
"rollingInterval": "Day",
"retainedFileCountLimit": 30,
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext} {Message:lj} {Properties:j}{NewLine}{Exception}"
}
},
{
"Name": "Seq",
"Args": {
"serverUrl": "http://localhost:5341",
"restrictedToMinimumLevel": "Information"
}
}
],
"Enrich": ["FromLogContext", "WithMachineName", "WithThreadId"]
},
"AllowedHosts": "*",
"ConnectionStrings": {
"SQLDatabase": "#{db-connection-string}#"

13
src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase.json
View File

@@ -1,13 +0,0 @@
{
"type": "service_account",
"project_id": "#{firebase-project-id}#",
"private_key_id": "#{firebase-private-key-id}#",
"private_key": "#{firebase-private-key}#",
"client_email": "#{firebase-client-email}#",
"client_id": "#{firebase-client-id}#",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "#{firebase-client-cert-url}#",
"universe_domain": "googleapis.com"
}