diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 190f273..059641a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,6 @@ jobs: run: | rm -f appsettings.Development.json rm -f client_secrets.Development.json - rm -f diunabi-admin-firebase-Development.json - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/buildScripts/replaceTokens.js b/.github/workflows/buildScripts/replaceTokens.js index 17808b0..565cfdb 100644 --- a/.github/workflows/buildScripts/replaceTokens.js +++ b/.github/workflows/buildScripts/replaceTokens.js @@ -10,7 +10,6 @@ module.exports = async ({ github, context, core, jobId }) => { files.push(`./${jobId}/webapi/appsettings.json`); files.push(`./${jobId}/webapi/client_secrets.json`); - files.push(`./${jobId}/webapi/diunabi-admin-firebase.json`); files.forEach(file => { let data = require('fs').readFileSync(file, 'utf8'); diff --git a/.gitignore b/.gitignore index 2ecb198..9525dea 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,4 @@ Temp/ # Sensitive files **/appsettings.Development.json -**/firebase-adminsdk-*.json -src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase-Development.json diff --git a/src/Backend/DiunaBI.Core/Models/LogEntry.cs b/src/Backend/DiunaBI.Core/Models/LogEntry.cs deleted file mode 100644 index e9aa3e9..0000000 --- a/src/Backend/DiunaBI.Core/Models/LogEntry.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; - -namespace DiunaBI.Core.Models; - -public enum LogEntryType -{ - Info, - Warning, - Error -} -public enum LogType -{ - Import, - Backup, - Process, - PowerBi, - DataInbox, - Queue -} - -public enum LogInstance -{ - Morska -} -public class LogEntry -{ - public LogType LogType { get; init; } - public LogEntryType Type { get; init; } - public string? Message { get; init; } - public string? Title { get; init; } - public DateTime CreatedAt { get; init; } - public Guid SessionId { get; set; } - public LogInstance Instance { get; set; } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.Core/Services/PluginManager.cs b/src/Backend/DiunaBI.Core/Services/PluginManager.cs index a9119a6..0a0d8b4 100644 --- a/src/Backend/DiunaBI.Core/Services/PluginManager.cs +++ b/src/Backend/DiunaBI.Core/Services/PluginManager.cs @@ -46,8 +46,10 @@ public class PluginManager } } - _logger.LogInformation("Loaded {ProcessorCount} processors and {ImporterCount} importers from {PluginCount} plugins", - _processorTypes.Count, _importerTypes.Count, _plugins.Count); + _logger.LogInformation("Loaded {ProcessorCount} processors and {ImporterCount} importers from {AssemblyCount} assemblies", + _processorTypes.Count, + _importerTypes.Count, + dllFiles.Length); // Zmień z _plugins.Count na assemblyFiles.Length } private void LoadPluginFromAssembly(string assemblyPath) diff --git a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj index 5fbd6b8..f208118 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj +++ b/src/Backend/DiunaBI.Plugins.Morska/DiunaBI.Plugins.Morska.csproj @@ -6,15 +6,12 @@ - - + + - - - - + + - \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs index da60532..677ec5a 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/MorskaBaseProcessor.cs @@ -1,5 +1,6 @@ using DiunaBI.Core.Interfaces; using DiunaBI.Core.Models; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs index 61589e2..9ef9407 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R1Processor.cs @@ -6,6 +6,7 @@ using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services; using DiunaBI.Core.Services.Calculations; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; @@ -15,12 +16,16 @@ public class T1R1Processor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T1R1Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } public override void Process(Layer processWorker) { @@ -108,34 +113,16 @@ public class T1R1Processor : MorskaBaseProcessor { if (dynamicCode.Desc1 == null) { - //TODO throw exception or log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula in Record {dynamicCode.Id} is missing.", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T1R1: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } var calc = new BaseCalc(dynamicCode.Desc1); if (!calc.IsFormulaCorrect()) { - //TODO throw exception or log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T1R1: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } @@ -145,33 +132,14 @@ public class T1R1Processor : MorskaBaseProcessor } catch (Exception e) { - //TODO throw exception or log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = - $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T1R1: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); } } catch (Exception e) { - //TODO throw exception or log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T1R1: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); } } } @@ -195,8 +163,7 @@ public class T1R1Processor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: Save records to the layer - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); var sheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; @@ -208,6 +175,27 @@ public class T1R1Processor : MorskaBaseProcessor UpdateReport(processedLayer.Id, sheetName); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } + private void UpdateReport(Guid sourceId, string sheetName) { const string sheetId = "1pph-XowjlK5CIaCEV_A5buK4ceJ0Z0YoUlDI4VMkhhA"; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs index 4d5139f..6a6f633 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T1R3Processor.cs @@ -6,6 +6,7 @@ using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; @@ -15,12 +16,16 @@ public class T1R3Processor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T1R3Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } public override void Process(Layer processWorker) { @@ -108,13 +113,34 @@ public class T1R3Processor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); UpdateReport(processedLayer.Id, year); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } + private void UpdateReport(Guid sourceId, int year) { const string sheetId = "10Xo8BBF92nM7_JzzeOuWp49Gz8OsYuCxLDOeChqpW_8"; diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs index bc998ca..c35f36c 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesProcessor.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; @@ -10,10 +11,14 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor public override string ProcessorType => "T3.MultiSourceCopySelectedCodes"; private readonly AppDbContext _db; + private readonly ILogger _logger; + public T3MultiSourceCopySelectedCodesProcessor( - AppDbContext db) + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } public override void Process(Layer processWorker) { @@ -98,8 +103,29 @@ public class T3MultiSourceCopySelectedCodesProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: Save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs index 4eb87a6..d54728b 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceCopySelectedCodesYearSummaryProcessor.cs @@ -2,18 +2,23 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.MultiSourceCopySelectedCodesYearSummary"; + private readonly AppDbContext _db; + private readonly ILogger _logger; public T3MultiSourceCopySelectedCodesYearSummaryProcessor( - AppDbContext db) + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } public override void Process(Layer processWorker) { @@ -100,8 +105,27 @@ public class T3MultiSourceCopySelectedCodesYearSummaryProcessor : MorskaBaseProc { _db.Layers.Update(processedLayer); } - //TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs index 2eba5ff..b4a7cb7 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceSummaryProcessor.cs @@ -1,5 +1,6 @@ using DiunaBI.Core.Services; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services.Calculations; @@ -9,15 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.MultiSourceSummary"; + private readonly AppDbContext _db; + private readonly ILogger _logger; - T3MultiSourceSummaryProcessor( - AppDbContext db) + public T3MultiSourceSummaryProcessor( + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } + public override void Process(Layer processWorker) { + _logger.LogInformation("T3MultiSourceSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); @@ -72,7 +81,6 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode) .ToList(); @@ -95,6 +103,7 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor var dynamicCodes = processWorker.Records? .Where(x => x.Code!.Contains("DynamicCode-")) .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); + if (dynamicCodes != null && dynamicCodes.Count != 0) { foreach (var dynamicCode in dynamicCodes) @@ -103,33 +112,16 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor { if (dynamicCode.Desc1 == null) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula in Record {dynamicCode.Id} is missing.", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T3MultiSourceSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } + var calc = new BaseCalc(dynamicCode.Desc1); if (!calc.IsFormulaCorrect()) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T3MultiSourceSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } @@ -139,37 +131,18 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor } catch (Exception e) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T3MultiSourceSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); } } catch (Exception e) { - // TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T3MultiSourceSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); } } } - if (isNew) { _db.Layers.Add(processedLayer); @@ -178,8 +151,32 @@ public class T3MultiSourceSummaryProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogInformation("T3MultiSourceSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs index 8f98381..90f7361 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3MultiSourceYearSummaryProcessor.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services; @@ -9,14 +10,23 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.MultiSourceYearSummary"; + private readonly AppDbContext _db; - T3MultiSourceYearSummaryProcessor( - AppDbContext db) + private readonly ILogger _logger; + + public T3MultiSourceYearSummaryProcessor( + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } + public override void Process(Layer processWorker) { + _logger.LogInformation("T3MultiSourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); if (sources!.Count == 0) @@ -59,23 +69,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor .FirstOrDefault()) .OfType() .ToList(); + if (dataSources.Count == 0) { throw new Exception("DataSources are empty"); } - if (dataSources.Count == 0) - { - throw new Exception("DataSourcesValidation are empty"); - } - var allRecords = dataSources .SelectMany(x => x.Records!).ToList(); var baseCodes = allRecords.Select(x => x.Code!.Remove(0, 1)).Distinct().ToList(); foreach (var baseCode in baseCodes) { - var codeRecords = allRecords.Where(x => x.Code![1..] == baseCode) .ToList(); @@ -110,12 +115,11 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor newRecords.Add(processedRecord); } - - // Dynamic Codes var dynamicCodes = processWorker.Records? .Where(x => x.Code!.Contains("DynamicCode-")) .OrderBy(x => int.Parse(x.Code!.Split('-')[1])).ToList(); + if (dynamicCodes != null && dynamicCodes.Count != 0) { foreach (var dynamicCode in dynamicCodes) @@ -124,33 +128,16 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor { if (dynamicCode.Desc1 == null) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula in Record {dynamicCode.Id} is missing.", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T3MultiSourceYearSummary: Formula in Record {RecordId} is missing. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } + var calc = new BaseCalc(dynamicCode.Desc1); if (!calc.IsFormulaCorrect()) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} is not correct", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} is not correct. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); continue; } @@ -160,35 +147,18 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor } catch (Exception e) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Formula {calc.Expression} in Record {dynamicCode.Id} error: {e.Message}", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T3MultiSourceYearSummary: Formula {Expression} in Record {RecordId} calculation error. Process: {ProcessName} ({ProcessId})", + calc.Expression, dynamicCode.Id, processWorker.Name, processWorker.Id); } } catch (Exception e) { - //TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Calculation error {dynamicCode.Id}: {e.Message} ", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning(e, "T3MultiSourceYearSummary: Calculation error for DynamicCode {RecordId}. Process: {ProcessName} ({ProcessId})", + dynamicCode.Id, processWorker.Name, processWorker.Id); } } } + if (isNew) { _db.Layers.Add(processedLayer); @@ -197,8 +167,32 @@ public class T3MultiSourceYearSummaryProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogInformation("T3MultiSourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs index 0de1bef..0573f63 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SingleSourceProcessor.cs @@ -2,17 +2,27 @@ using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; namespace DiunaBI.Plugins.Morska.Processors; public class T3SingleSourceProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.SingleSource"; + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T3SingleSourceProcessor( - AppDbContext db) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; } public override void Process(Layer processWorker) { @@ -139,9 +149,28 @@ public class T3SingleSourceProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs index 88d1e08..85354e9 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T3SourceYearSummaryProcessor.cs @@ -1,4 +1,6 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services; @@ -8,20 +10,33 @@ namespace DiunaBI.Plugins.Morska.Processors; public class T3SourceYearSummaryProcessor : MorskaBaseProcessor { public override string ProcessorType => "T3.SourceYearSummary"; + private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T3SourceYearSummaryProcessor( - AppDbContext db) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; } + public override void Process(Layer processWorker) { + _logger.LogInformation("T3SourceYearSummary: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; if (source == null) { throw new Exception("Source record not found"); } + var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) @@ -101,8 +116,32 @@ public class T3SourceYearSummaryProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - //TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogInformation("T3SourceYearSummary: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3SourceYearSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs index 10246fd..edb9e73 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4R2Processor.cs @@ -2,6 +2,7 @@ using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using DiunaBI.Core.Models; using DiunaBI.Database.Context; using DiunaBI.Core.Services; @@ -15,15 +16,23 @@ public class T4R2Processor : MorskaBaseProcessor private readonly AppDbContext _db; private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T4R2Processor( - AppDbContext db, - SpreadsheetsResource.ValuesResource googleSheetValues) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; _googleSheetValues = googleSheetValues; + _logger = logger; } + public override void Process(Layer processWorker) { + _logger.LogInformation("T4R2: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var sources = processWorker.Records?.Where(x => x.Code == "Source").ToList(); if (sources!.Count == 0) @@ -37,7 +46,6 @@ public class T4R2Processor : MorskaBaseProcessor throw new Exception("LayerName record not found"); } - var processedLayer = _db.Layers .Where(x => x.ParentId == processWorker.Id && !x.IsDeleted && !x.IsCancelled) @@ -116,17 +124,8 @@ public class T4R2Processor : MorskaBaseProcessor } else { - // TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Data source {year}/{month:D2}-{source.Desc1}-T3 not found", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", + $"{year}/{month:D2}-{source.Desc1}-T3", processWorker.Name, processWorker.Id); } } else @@ -182,17 +181,8 @@ public class T4R2Processor : MorskaBaseProcessor } else { - // TODO: log warning - /* - logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Process, - Message = $"Data source {year}/13-{source.Desc1}-T3 not found", - CreatedAt = DateTime.UtcNow - }); - */ + _logger.LogWarning("T4R2: Data source {DataSource} not found. Process: {ProcessName} ({ProcessId})", + $"{year}/13-{source.Desc1}-T3", processWorker.Name, processWorker.Id); } } @@ -204,8 +194,8 @@ public class T4R2Processor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - // TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); var reportSheetName = processWorker.Records?.SingleOrDefault(x => x.Code == "GoogleSheetName")?.Desc1; @@ -219,7 +209,32 @@ public class T4R2Processor : MorskaBaseProcessor { throw new Exception("GoogleSheetName-Invoices record not found"); } + UpdateReport(processedLayer.Id, reportSheetName, invoicesSheetName); + + _logger.LogInformation("T4R2: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + } + + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T4R2: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); } private void UpdateReport(Guid sourceId, string reportSheetName, string invoicesSheetName) diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs index f7c0c12..c919523 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T4SingleSourceProcessor.cs @@ -1,6 +1,8 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; +using Microsoft.Extensions.Logging; +using Google.Apis.Sheets.v4; namespace DiunaBI.Plugins.Morska.Processors; @@ -9,13 +11,24 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor public override string ProcessorType => "T4.SingleSource"; private readonly AppDbContext _db; + private readonly SpreadsheetsResource.ValuesResource _googleSheetValues; + private readonly ILogger _logger; + public T4SingleSourceProcessor( - AppDbContext db) + AppDbContext db, + SpreadsheetsResource.ValuesResource googleSheetValues, + ILogger logger) { _db = db; + _googleSheetValues = googleSheetValues; + _logger = logger; } + public override void Process(Layer processWorker) { + _logger.LogInformation("T4SingleSource: Starting processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); + var year = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1!); var month = int.Parse(processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1!); var sourceLayer = processWorker.Records?.SingleOrDefault(x => x.Code == "SourceLayer")?.Desc1; @@ -23,13 +36,15 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor { throw new Exception("SourceLayer record not found"); } + var sourceImportWorker = _db.Layers.SingleOrDefault(x => x.Name == sourceLayer && !x.IsDeleted && !x.IsCancelled); if (sourceImportWorker == null) { - throw new Exception("SourceImportWorkerL layer not found"); + throw new Exception("SourceImportWorker layer not found"); } + var source = processWorker.Records?.SingleOrDefault(x => x.Code == "Source")?.Desc1; - if (sourceLayer == null) + if (source == null) { throw new Exception("Source record not found"); } @@ -61,7 +76,6 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor processedLayer.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); processedLayer.ModifiedAt = DateTime.UtcNow; - var dataSource = _db.Layers .Include(x => x.Records) .Where(x => x.ParentId == sourceImportWorker.Id @@ -83,8 +97,7 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor Value1 = record.Value1, CreatedAt = DateTime.UtcNow, ModifiedAt = DateTime.UtcNow - }) - .ToList(); + }).ToList(); if (isNew) { @@ -94,9 +107,32 @@ public class T4SingleSourceProcessor : MorskaBaseProcessor { _db.Layers.Update(processedLayer); } - // TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); + + _logger.LogInformation("T4SingleSource: Successfully completed processing for {ProcessWorkerName} ({ProcessWorkerId})", + processWorker.Name, processWorker.Id); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T4SingleSource: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs index 56a78d6..5b3bc56 100644 --- a/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs +++ b/src/Backend/DiunaBI.Plugins.Morska/Processors/T5LastValuesProcessor.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using DiunaBI.Core.Models; using DiunaBI.Database.Context; +using Microsoft.Extensions.Logging; namespace DiunaBI.Plugins.Morska.Processors; @@ -10,10 +11,14 @@ public class T5LastValuesProcessor : MorskaBaseProcessor public override string ProcessorType => "T5.LastValues"; private readonly AppDbContext _db; + private readonly ILogger _logger; + public T5LastValuesProcessor( - AppDbContext db) + AppDbContext db, + ILogger logger) { _db = db; + _logger = logger; } public override void Process(Layer processWorker) { @@ -95,8 +100,27 @@ public class T5LastValuesProcessor : MorskaBaseProcessor _db.Layers.Add(processedLayer); else _db.Layers.Update(processedLayer); - // TODO: save records - //controller.SaveRecords(processedLayer.Id, newRecords, Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")); + SaveRecords(processedLayer.Id, newRecords); _db.SaveChanges(); } + private void SaveRecords(Guid layerId, ICollection records) + { + var toDelete = _db.Records.Where(x => x.LayerId == layerId).ToList(); + if (toDelete.Count > 0) + { + _db.Records.RemoveRange(toDelete); + } + + foreach (var record in records) + { + record.CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.CreatedAt = DateTime.UtcNow; + record.ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"); + record.ModifiedAt = DateTime.UtcNow; + record.LayerId = layerId; + _db.Records.Add(record); + } + + _logger.LogDebug("T3MultiSourceSummary: Saved {RecordCount} records for layer {LayerId}", records.Count, layerId); + } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs index 569495a..10ad60f 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/DataInboxController.cs @@ -2,8 +2,8 @@ using System.Text; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using DiunaBI.Database.Context; -using Google.Cloud.Firestore; using DiunaBI.Core.Models; namespace DiunaBI.WebAPI.Controllers; @@ -14,16 +14,16 @@ public class DataInboxController : Controller { private readonly AppDbContext _db; private readonly IConfiguration _configuration; - private readonly LogsController _logsController; + private readonly ILogger _logger; public DataInboxController( AppDbContext db, IConfiguration configuration, - FirestoreDb firestoreDb) + ILogger logger) { _db = db; _configuration = configuration; - _logsController = new LogsController(firestoreDb); + _logger = logger; } [HttpPut] @@ -33,41 +33,22 @@ public class DataInboxController : Controller { if (apiKey != _configuration["apiKey"]) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - wrong apiKey ({dataInbox.Source})", - Type = LogEntryType.Warning, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("DataInbox: Unauthorized request - wrong apiKey for source {Source}", dataInbox.Source); return Unauthorized(); } try { - if ( - !Request.Headers.TryGetValue("Authorization", out var authHeader)) + if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - no authorization header ({dataInbox.Source})", - Type = LogEntryType.Warning, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("DataInbox: Unauthorized request - no authorization header for source {Source}", dataInbox.Source); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - wrong auth header format ({dataInbox.Source})", - Type = LogEntryType.Warning, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("DataInbox: Unauthorized request - wrong auth header format for source {Source}", dataInbox.Source); return Unauthorized(); } @@ -76,60 +57,35 @@ public class DataInboxController : Controller var password = authValue.Split(':')[1]; if (username != _configuration["morska-user"] || password != _configuration["morska-pass"]) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - bad credentials ({dataInbox.Source})", - Type = LogEntryType.Warning, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("DataInbox: Unauthorized request - bad credentials for source {Source}", dataInbox.Source); return Unauthorized(); } // check if datainbox.data is base64 encoded value if (!string.IsNullOrEmpty(dataInbox.Data) && !IsBase64String(dataInbox.Data)) { - _logsController.AddEntry(new LogEntry - { - Title = $"Invalid data format - not base64 encoded ({dataInbox.Source})", - Type = LogEntryType.Warning, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("DataInbox: Invalid data format - not base64 encoded for source {Source}", dataInbox.Source); return BadRequest("Invalid data format - not base64 encoded"); } - dataInbox.Id = Guid.NewGuid(); dataInbox.CreatedAt = DateTime.UtcNow; _db.DataInbox.Add(dataInbox); _db.SaveChanges(); - _logsController.AddEntry(new LogEntry - { - Title = $"Insert success: {dataInbox.Source}, {dataInbox.Name}", - Type = LogEntryType.Info, - LogType = LogType.DataInbox, - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("DataInbox: Insert success for source {Source}, name {Name}", dataInbox.Source, dataInbox.Name); if (dataInbox.Name == "morska.d3.importer") { - // TODO: import dataInbox as Layer + _logger.LogDebug("DataInbox: Detected morska.d3.importer - processing will be handled by AutoImport"); + // AutoImport będzie obsługiwać ten typ danych } return Ok(); } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = $"Insert error: {dataInbox.Source}, {dataInbox.Name}", - Type = LogEntryType.Error, - LogType = LogType.DataInbox, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "DataInbox: Insert error for source {Source}, name {Name}", dataInbox.Source, dataInbox.Name); return BadRequest(e.ToString()); } } @@ -137,7 +93,17 @@ public class DataInboxController : Controller [HttpGet] public IActionResult GetAll() { - return Ok(_db.DataInbox.AsNoTracking().ToList()); + try + { + var dataInbox = _db.DataInbox.AsNoTracking().ToList(); + _logger.LogDebug("DataInbox: Retrieved {Count} records", dataInbox.Count); + return Ok(dataInbox); + } + catch (Exception e) + { + _logger.LogError(e, "DataInbox: Error retrieving records"); + return BadRequest(e.ToString()); + } } // helpers @@ -150,9 +116,7 @@ public class DataInboxController : Controller try { var base64Bytes = Convert.FromBase64String(data); - var utf8String = Encoding.UTF8.GetString(base64Bytes); - var reEncoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(utf8String)); return data.TrimEnd('=') == reEncoded.TrimEnd('='); } diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs index 19c6edf..8f67a2f 100644 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs +++ b/src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs @@ -19,28 +19,24 @@ public class LayersController : Controller private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues; private readonly GoogleDriveHelper _googleDriveHelper; private readonly IConfiguration _configuration; - private readonly LogsController _logsController; private readonly PluginManager _pluginManager; + private readonly ILogger _logger; public LayersController( AppDbContext db, - GoogleSheetsHelper googleSheetsHelper, + SpreadsheetsResource.ValuesResource? googleSheetValues, GoogleDriveHelper googleDriveHelper, IConfiguration configuration, - FirestoreDb firestoreDb, - PluginManager pluginManager + PluginManager pluginManager, + ILogger logger ) { _db = db; - if (googleSheetsHelper.Service is not null) - { - _googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values; - } - + _googleSheetValues = googleSheetValues; _googleDriveHelper = googleDriveHelper; _configuration = configuration; - _logsController = new LogsController(firestoreDb); _pluginManager = pluginManager; + _logger = logger; } [HttpGet] @@ -59,12 +55,18 @@ public class LayersController : Controller response = response.Where(x => x.Type == type); } - return Ok(response + var result = response .OrderByDescending(x => x.Number) - .Skip(start).Take(limit).AsNoTracking().ToList()); + .Skip(start).Take(limit).AsNoTracking().ToList(); + + _logger.LogDebug("GetAll: Retrieved {Count} layers with filter name={Name}, type={Type}", + result.Count, name, type); + + return Ok(result); } catch (Exception e) { + _logger.LogError(e, "GetAll: Error retrieving layers"); return BadRequest(e.ToString()); } } @@ -75,13 +77,17 @@ public class LayersController : Controller { try { - return Ok(_db.Layers + var layer = _db.Layers .Include(x => x.CreatedBy) .Include(x => x.ModifiedBy) - .Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted)); + .Include(x => x.Records).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); + + _logger.LogDebug("Get: Retrieved layer {LayerId} {LayerName}", id, layer.Name); + return Ok(layer); } catch (Exception e) { + _logger.LogError(e, "Get: Error retrieving layer {LayerId}", id); return BadRequest(e.ToString()); } } @@ -92,41 +98,22 @@ public class LayersController : Controller { if (apiKey != _configuration["apiKey"]) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - wrong apiKey ({number})", - Type = LogEntryType.Warning, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("PowerBI: Unauthorized request - wrong apiKey for layer {LayerNumber}", number); return Unauthorized(); } try { - if ( - !Request.Headers.TryGetValue("Authorization", out var authHeader)) + if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - no authorization header ({number})", - Type = LogEntryType.Warning, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("PowerBI: Unauthorized request - no authorization header for layer {LayerNumber}", number); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - wrong auth header format ({number})", - Type = LogEntryType.Warning, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("PowerBI: Unauthorized request - wrong auth header format for layer {LayerNumber}", number); return Unauthorized(); } @@ -135,37 +122,21 @@ public class LayersController : Controller var password = authValue.Split(':')[1]; if (username != _configuration["powerBI-user"] || password != _configuration["powerBI-pass"]) { - _logsController.AddEntry(new LogEntry - { - Title = $"Unauthorized request - bad credentials ({number})", - Type = LogEntryType.Warning, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogWarning("PowerBI: Unauthorized request - bad credentials for layer {LayerNumber}", number); return Unauthorized(); } - _logsController.AddEntry(new LogEntry - { - Title = $"Sending data for layer {number}", - Type = LogEntryType.Info, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("PowerBI: Sending data for layer {LayerNumber}", number); - return Ok(_db.Layers + var layer = _db.Layers .Include(x => x.CreatedBy) - .Include(x => x.Records).AsNoTracking().First(x => x.Number == number && !x.IsDeleted)); + .Include(x => x.Records).AsNoTracking().First(x => x.Number == number && !x.IsDeleted); + + return Ok(layer); } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = e.ToString(), - Type = LogEntryType.Error, - LogType = LogType.PowerBi, - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "PowerBI: Error occurred while processing layer {LayerNumber}", number); return BadRequest(e.ToString()); } } @@ -176,20 +147,22 @@ public class LayersController : Controller { if (apiKey != _configuration["apiKey"]) { + _logger.LogWarning("Configuration: Unauthorized request - wrong apiKey for layer {LayerNumber}", number); return Unauthorized(); } try { - if ( - !Request.Headers.TryGetValue("Authorization", out var authHeader)) + if (!Request.Headers.TryGetValue("Authorization", out var authHeader)) { + _logger.LogWarning("Configuration: Unauthorized request - no authorization header for layer {LayerNumber}", number); return Unauthorized(); } var credentialsArr = authHeader.ToString().Split(" "); if (credentialsArr.Length != 2) { + _logger.LogWarning("Configuration: Unauthorized request - wrong auth header format for layer {LayerNumber}", number); return Unauthorized(); } @@ -198,6 +171,7 @@ public class LayersController : Controller var password = authValue.Split(':')[1]; if (username != _configuration["morska-user"] || password != _configuration["morska-pass"]) { + _logger.LogWarning("Configuration: Unauthorized request - bad credentials for layer {LayerNumber}", number); return Unauthorized(); } @@ -208,19 +182,23 @@ public class LayersController : Controller if (config is null) { + _logger.LogWarning("Configuration: Layer {LayerNumber} not found", number); return BadRequest(); } var type = config.Records?.Where(x => x.Code == "Type").FirstOrDefault(); if (type is null || type.Desc1 != "ExternalConfiguration") { + _logger.LogWarning("Configuration: Layer {LayerNumber} is not ExternalConfiguration type", number); return BadRequest(); } + _logger.LogInformation("Configuration: Sending configuration for layer {LayerNumber}", number); return Ok(config); } - catch + catch (Exception e) { + _logger.LogError(e, "Configuration: Error occurred while processing layer {LayerNumber}", number); return BadRequest(); } } @@ -231,19 +209,33 @@ public class LayersController : Controller { if (_googleSheetValues is null) { + _logger.LogError("Export: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } - var layer = _db.Layers - .Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); - - var export = _pluginManager.GetExporter("GoogleSheet"); - if (export == null) + try { - throw new Exception("GoogleSheet exporter not found"); + var layer = _db.Layers + .Include(x => x.Records!.OrderByDescending(y => y.Code)).AsNoTracking().First(x => x.Id == id && !x.IsDeleted); + + var export = _pluginManager.GetExporter("GoogleSheet"); + if (export == null) + { + _logger.LogError("Export: GoogleSheet exporter not found for layer {LayerId}", id); + throw new Exception("GoogleSheet exporter not found"); + } + + _logger.LogInformation("Export: Starting GoogleSheet export for layer {LayerId} {LayerName}", id, layer.Name); + export.Export(layer); + _logger.LogInformation("Export: Successfully exported layer {LayerId} to GoogleSheet", id); + + return Ok(true); + } + catch (Exception e) + { + _logger.LogError(e, "Export: Failed to export layer {LayerId} to GoogleSheet", id); + throw; } - export.Export(layer); - return Ok(true); } [HttpGet] @@ -253,6 +245,7 @@ public class LayersController : Controller { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { + _logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } @@ -268,38 +261,23 @@ public class LayersController : Controller if (importWorkerLayers.Count == 0) { - _logsController.AddEntry(new LogEntry - { - Title = "No Layers to import.", - Type = LogEntryType.Info, - LogType = LogType.Queue, - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("AutoImportQueue: No layers to import"); return Ok(); } + _logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count); + foreach (var importWorker in importWorkerLayers) { try { - /* - await _queue.AddJob(new QueueJob - { - LayerId = importWorker.Id, - Type = JobType.ImportWorker, - }); - */ + // Queue job implementation would go here + _logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id); } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = $"Error while adding job into queue (import): {importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Error, - LogType = LogType.Queue, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); } } return Ok(); @@ -310,17 +288,14 @@ public class LayersController : Controller [AllowAnonymous] public IActionResult ProcessQueue(string apiKey) { - /* - var allJobs = await _queue.GetJobs(); - var importJobs = allJobs - .Where(x => x.Type == JobType.ImportWorker && x.Status == JobStatus.New); - - foreach (var job in importJobs) + if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { - job.Attempts = job.Attempts + 1; - //await _queue.UpdateJob(job); + _logger.LogWarning("ProcessQueue: Unauthorized request with apiKey {ApiKey}", apiKey); + return Unauthorized(); } - */ + + _logger.LogInformation("ProcessQueue: Starting queue processing"); + // Queue processing implementation would go here return Ok(); } @@ -331,11 +306,13 @@ public class LayersController : Controller { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { + _logger.LogWarning("AutoImport: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { + _logger.LogError("AutoImport: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } @@ -349,24 +326,15 @@ public class LayersController : Controller .OrderByDescending(x => x.CreatedAt) .AsNoTracking() .ToList(); - _logsController.AddEntry(new LogEntry - { - Title = $"Starting import: {nameFilter}, Admin layers count ({importWorkerLayers.Count})", - Type = LogEntryType.Info, - LogType = LogType.Import, - CreatedAt = DateTime.UtcNow - }); + + _logger.LogInformation("AutoImport: Starting import with filter {NameFilter}, found {LayerCount} layers", + nameFilter, importWorkerLayers.Count); + try { if (importWorkerLayers.Count == 0) { - _logsController.AddEntry(new LogEntry - { - Title = "No Layers to import.", - Type = LogEntryType.Info, - LogType = LogType.Import, - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("AutoImport: No layers to import"); return Ok(); } @@ -374,10 +342,12 @@ public class LayersController : Controller { try { - var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? - "Standard"; - var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? - "GoogleSheet"; + var type = importWorker.Records!.FirstOrDefault(x => x.Code == "ImportType")?.Desc1 ?? "Standard"; + var source = importWorker.Records!.FirstOrDefault(x => x.Code == "Source")?.Desc1 ?? "GoogleSheet"; + + _logger.LogInformation("AutoImport: Processing layer {LayerName} with type {ImportType} and source {Source}", + importWorker.Name, type, source); + if (source == "DataInbox" && type == "Import-D3") { var d3Importer = _pluginManager.GetImporter("MorskaD3"); @@ -387,16 +357,11 @@ public class LayersController : Controller } d3Importer.Import(importWorker); - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Import, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("AutoImport: Successfully processed D3 import for {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); continue; } + switch (type) { case "D1": @@ -406,134 +371,88 @@ public class LayersController : Controller throw new Exception("MorskaD1 importer not found"); } d1importer.Import(importWorker); - Thread.Sleep(5000); // be aware of GSheet API quota - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Import, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); + _logger.LogInformation("AutoImport: Successfully processed D1 import for {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); break; + case "FK2": + var fk2importer = _pluginManager.GetImporter("MorskaFK2"); + if (fk2importer == null) { - var fk2importer = _pluginManager.GetImporter("MorskaFK2"); - if (fk2importer == null) + throw new Exception("MorskaFK2 importer not found"); + } + fk2importer.Import(importWorker); + Thread.Sleep(5000); // be aware of GSheet API quota + + _logger.LogInformation("AutoImport: Successfully processed FK2 import for {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); + break; + + default: + var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; + if (startDate == null) + { + throw new Exception("StartDate record not found"); + } + + var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; + if (endDate == null) + { + throw new Exception("EndDate record not found"); + } + + var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); + var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); + + if (startDateParsed.Date <= DateTime.UtcNow.Date && endDateParsed.Date >= DateTime.UtcNow.Date) + { + var importer = _pluginManager.GetImporter("MorskaImporter"); + if (importer == null) { - throw new Exception("MorskaFK2 importer not found"); + throw new Exception("MorskaImporter not found"); } - fk2importer.Import(importWorker); + importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Import, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - break; + _logger.LogInformation("AutoImport: Successfully processed standard import for {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); } - default: + else if (IsImportedLayerUpToDate(importWorker) == false) { - var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; - if (startDate == null) + var importer = _pluginManager.GetImporter("MorskaImporter"); + if (importer == null) { - throw new Exception("StartDate record nod found"); + throw new Exception("MorskaImporter not found"); } + importer.Import(importWorker); + Thread.Sleep(5000); // be aware of GSheet API quota - var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; - if (endDate == null) - { - throw new Exception("EndDate record nod found"); - } - - var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); - var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - if (startDateParsed.Date <= DateTime.UtcNow.Date && - endDateParsed.Date >= DateTime.UtcNow.Date) - { - var importer = _pluginManager.GetImporter("MorskaImporter"); - if (importer == null) - { - throw new Exception("MorskaImporter not found"); - } - importer.Import(importWorker); - Thread.Sleep(5000); // be aware of GSheet API quota - - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Import, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - } - else if (IsImportedLayerUpToDate(importWorker) == false) - { - var importer = _pluginManager.GetImporter("MorskaImporter"); - if (importer == null) - { - throw new Exception("MorskaImporter not found"); - } - importer.Import(importWorker); - Thread.Sleep(5000); // be aware of GSheet API quota - - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Import, - Message = "Success (reimported)", - CreatedAt = DateTime.UtcNow - }); - } - else - { - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Import, - Message = "importLayer records are up to date. Not processed.", - CreatedAt = DateTime.UtcNow - }); - } - - break; + _logger.LogWarning("AutoImport: Reimported out-of-date layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); } + else + { + _logger.LogInformation("AutoImport: Layer {LayerName} ({LayerId}) is up to date, skipping", + importWorker.Name, importWorker.Id); + } + break; } } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Error, - LogType = LogType.Import, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "AutoImport: Failed to process layer {LayerName} ({LayerId})", + importWorker.Name, importWorker.Id); } } + _logger.LogInformation("AutoImport: Completed processing {LayerCount} layers", importWorkerLayers.Count); return Ok(); } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = "Process error", - Type = LogEntryType.Error, - LogType = LogType.Import, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "AutoImport: Process error"); return BadRequest(e.ToString()); } } @@ -545,11 +464,13 @@ public class LayersController : Controller { if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"]) { + _logger.LogWarning("AutoProcess: Unauthorized request with apiKey {ApiKey}", apiKey); return Unauthorized(); } if (_googleSheetValues is null) { + _logger.LogError("AutoProcess: Google Sheets API not initialized"); throw new Exception("Google Sheets API not initialized"); } @@ -557,8 +478,8 @@ public class LayersController : Controller [ "T3-SingleSource", "T3-SourceYearSummary", - "T3-MultiSourceSummary", // AA - "T3-MultiSourceYearSummary", // AA/13 + "T3-MultiSourceSummary", + "T3-MultiSourceYearSummary", "T4-SingleSource", "T5-LastValues", "T1-R1", @@ -566,6 +487,8 @@ public class LayersController : Controller "T1-R3" ]; + _logger.LogInformation("AutoProcess: Starting processing for {ProcessTypeCount} process types", processTypes.Length); + foreach (var type in processTypes) { try @@ -581,38 +504,31 @@ public class LayersController : Controller .AsNoTracking() .ToList(); + _logger.LogInformation("AutoProcess: Processing type {ProcessType}, found {LayerCount} layers", + type, processWorkerLayers.Count); + foreach (var processWorker in processWorkerLayers) { try { ProcessLayer(processWorker); + _logger.LogInformation("AutoProcess: Successfully processed {LayerName} ({LayerId}) with type {ProcessType}", + processWorker.Name, processWorker.Id, type); } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Error, - LogType = LogType.Process, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "AutoProcess: Failed to process {LayerName} ({LayerId}) with type {ProcessType}", + processWorker.Name, processWorker.Id, type); } } } catch (Exception e) { - _logsController.AddEntry(new LogEntry - { - Title = "Process error", - Type = LogEntryType.Error, - LogType = LogType.Process, - Message = e.ToString(), - CreatedAt = DateTime.UtcNow - }); + _logger.LogError(e, "AutoProcess: Error processing type {ProcessType}", type); } } + _logger.LogInformation("AutoProcess: Completed processing all process types"); return Ok(); } @@ -626,7 +542,7 @@ public class LayersController : Controller var year = processWorker.Records?.SingleOrDefault(x => x.Code == "Year")?.Desc1; if (year == null) { - throw new Exception("Year record nod found"); + throw new Exception("Year record not found"); } var processType = processWorker.Records?.SingleOrDefault(x => x.Code == "ProcessType")?.Desc1; @@ -642,15 +558,6 @@ public class LayersController : Controller throw new Exception("T3.SourceYearSummary processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } case "T3-MultiSourceYearSummary": @@ -661,15 +568,6 @@ public class LayersController : Controller throw new Exception("T3.MultiSourceYearSummary processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } case "T3-MultiSourceCopySelectedCodesYearSummary": @@ -680,15 +578,6 @@ public class LayersController : Controller throw new Exception("T3.MultiSourceCopySelectedCodesYearSummary processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } case "T1-R1": @@ -699,15 +588,6 @@ public class LayersController : Controller throw new Exception("T1.R1 processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } case "T4-R2": @@ -718,15 +598,6 @@ public class LayersController : Controller throw new Exception("T4.R2 processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } case "T1-R3": @@ -737,15 +608,6 @@ public class LayersController : Controller throw new Exception("T1.R3 processor not found"); } processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); return; } } @@ -809,35 +671,8 @@ public class LayersController : Controller break; } } - - _logsController.AddEntry(new LogEntry - { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); } - internal void SaveRecords(Guid id, ICollection records, Guid currentUserId) - { - var toDelete = _db.Records.Where(x => x.LayerId == id).ToList(); - if (toDelete.Count > 0) - { - _db.Records.RemoveRange(toDelete); - } - - foreach (var record in records) - { - record.CreatedById = currentUserId; - record.CreatedAt = DateTime.UtcNow; - record.ModifiedById = currentUserId; - record.ModifiedAt = DateTime.UtcNow; - record.LayerId = id; - _db.Records.Add(record); - } - } private static void WriteToConsole(params string[] messages) { @@ -863,6 +698,7 @@ public class LayersController : Controller if (newestLayer is null) { + _logger.LogDebug("IsImportedLayerUpToDate: No child layers found for {LayerName}, treating as up to date", importWorker.Name); return true; // importWorker is not active yet, no check needed } @@ -884,39 +720,51 @@ public class LayersController : Controller throw new Exception($"DataRange not found, {importWorker.Name}"); } - var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); - var data = dataRangeResponse.Values; - - var isUpToDate = true; - - for (var i = 0; i < data[1].Count; i++) + try { - if (data[0][i].ToString() == "") continue; - var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); - if (record == null) + var dataRangeResponse = _googleSheetValues.Get(sheetId, $"{sheetTabName}!{dataRange}").Execute(); + var data = dataRangeResponse.Values; + + var isUpToDate = true; + + for (var i = 0; i < data[1].Count; i++) { - WriteToConsole("Code not found in DiunaBI", data[0][i].ToString()!); + if (data[0][i].ToString() == "") continue; + var record = newestLayer.Records!.FirstOrDefault(x => x.Code == data[0][i].ToString()); + if (record == null) + { + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in DiunaBI for layer {LayerName}", + data[0][i].ToString(), importWorker.Name); + isUpToDate = false; + continue; + } + + if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), out var value) || + double.Abs((double)(record.Value1 - value)!) < 0.01) continue; isUpToDate = false; - continue; } - if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"), - out var value) || - double.Abs((double)(record.Value1 - value)!) < 0.01) continue; - isUpToDate = false; - } - - foreach (var record in newestLayer.Records!) - { - if (data[0].Contains(record.Code)) + foreach (var record in newestLayer.Records!) { - continue; + if (data[0].Contains(record.Code)) + { + continue; + } + + _logger.LogDebug("IsImportedLayerUpToDate: Code {Code} not found in GoogleSheet for layer {LayerName}", + record.Code, importWorker.Name); + isUpToDate = false; } - WriteToConsole($"Code not found in GoogleSheet: {record.Code}"); - isUpToDate = false; - } + _logger.LogDebug("IsImportedLayerUpToDate: Layer {LayerName} is {Status}", + importWorker.Name, isUpToDate ? "up to date" : "outdated"); - return isUpToDate; + return isUpToDate; + } + catch (Exception e) + { + _logger.LogError(e, "IsImportedLayerUpToDate: Error checking if layer {LayerName} is up to date", importWorker.Name); + throw; + } } } \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/Controllers/LogsController.cs b/src/Backend/DiunaBI.WebAPI/Controllers/LogsController.cs deleted file mode 100644 index d98eae9..0000000 --- a/src/Backend/DiunaBI.WebAPI/Controllers/LogsController.cs +++ /dev/null @@ -1,46 +0,0 @@ -using Microsoft.AspNetCore.Mvc; -using DiunaBI.Core.Models; -using Google.Cloud.Firestore; - -namespace DiunaBI.WebAPI.Controllers; - -public class LogsController : Controller -{ - - private readonly FirestoreDb _firestoreDb; - private readonly Guid _SessionId = Guid.NewGuid(); - public LogsController( - FirestoreDb firestoreDb - ) - { - _firestoreDb = firestoreDb; - } - - public void AddEntry(LogEntry entry) - { - entry.SessionId = _SessionId; - entry.Instance = LogInstance.Morska; - - if (entry.Type == LogEntryType.Info) { return; } - - try - { - var collection = _firestoreDb.Collection("ApiLogs"); - var document = collection.Document(); - document.SetAsync(new - { - entry.Message, - entry.Title, - Type = Enum.GetName(typeof(LogEntryType), entry.Type), - LogType = Enum.GetName(typeof(LogType), entry.LogType), - Instance = Enum.GetName(typeof(LogInstance), entry.Instance), - entry.CreatedAt, - SessionId = entry.SessionId.ToString() - }).Wait(); - } - catch (Exception e) - { - Console.WriteLine(e.Message); - } - } -} \ No newline at end of file diff --git a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj index 70ccdda..6f45a65 100644 --- a/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj +++ b/src/Backend/DiunaBI.WebAPI/DiunaBI.WebAPI.csproj @@ -6,7 +6,6 @@ - diff --git a/src/Backend/DiunaBI.WebAPI/Program.cs b/src/Backend/DiunaBI.WebAPI/Program.cs index 98017b8..0d62ca9 100644 --- a/src/Backend/DiunaBI.WebAPI/Program.cs +++ b/src/Backend/DiunaBI.WebAPI/Program.cs @@ -1,4 +1,3 @@ -using FirebaseAdmin; using Google.Apis.Auth.OAuth2; using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authentication.JwtBearer; @@ -87,24 +86,10 @@ builder.Services.AddSingleton(provider => return valuesResource; }); - -var fileName = "diunabi-admin-firebase.json"; -#if DEBUG -fileName = "diunabi-admin-firebase-Development.json"; -#endif -var credentialPath = Path.Combine(Directory.GetCurrentDirectory(), fileName); -System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialPath); -FirebaseAdmin.FirebaseApp.Create(new AppOptions() -{ - Credential = GoogleCredential.GetApplicationDefault() -}); -builder.Services.AddSingleton(FirestoreDb.Create("diunabi-admin")); - builder.Services.AddSingleton(); var app = builder.Build(); -// ✅ DODAJ SERILOG REQUEST LOGGING app.UseSerilogRequestLogging(options => { options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; @@ -113,20 +98,17 @@ app.UseSerilogRequestLogging(options => diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value); diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme); - // ✅ POPRAW NULLABLE WARNING var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault(); if (!string.IsNullOrEmpty(userAgent)) { diagnosticContext.Set("UserAgent", userAgent); } - // Dodaj więcej użytecznych właściwości diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"); diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none"); }; }); -// Załaduj pluginy - z logowaniem var pluginManager = app.Services.GetRequiredService(); var executablePath = Assembly.GetExecutingAssembly().Location; var executableDir = Path.GetDirectoryName(executablePath)!; diff --git a/src/Backend/DiunaBI.WebAPI/bin/Debug/net8.0/appsettings.json b/src/Backend/DiunaBI.WebAPI/bin/Debug/net8.0/appsettings.json index 01e744b..89147f0 100644 --- a/src/Backend/DiunaBI.WebAPI/bin/Debug/net8.0/appsettings.json +++ b/src/Backend/DiunaBI.WebAPI/bin/Debug/net8.0/appsettings.json @@ -7,6 +7,44 @@ "Microsoft.AspNetCore": "Warning" } }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft.AspNetCore": "Warning", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning", + "Microsoft.EntityFrameworkCore.Infrastructure": "Warning", + "System.Net.Http.HttpClient": "Warning", + "Google.Apis": "Warning", + "DiunaBI.Core.Services.PluginManager": "Information" + } + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}" + } + }, + { + "Name": "File", + "Args": { + "path": "/var/log/diunabi/app-.log", + "rollingInterval": "Day", + "retainedFileCountLimit": 30, + "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level:u3}] {SourceContext} {Message:lj} {Properties:j}{NewLine}{Exception}" + } + }, + { + "Name": "Seq", + "Args": { + "serverUrl": "http://localhost:5341", + "restrictedToMinimumLevel": "Information" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithThreadId"] + }, "AllowedHosts": "*", "ConnectionStrings": { "SQLDatabase": "#{db-connection-string}#" diff --git a/src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase.json b/src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase.json deleted file mode 100644 index a27955f..0000000 --- a/src/Backend/DiunaBI.WebAPI/diunabi-admin-firebase.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "service_account", - "project_id": "#{firebase-project-id}#", - "private_key_id": "#{firebase-private-key-id}#", - "private_key": "#{firebase-private-key}#", - "client_email": "#{firebase-client-email}#", - "client_id": "#{firebase-client-id}#", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "#{firebase-client-cert-url}#", - "universe_domain": "googleapis.com" -}