mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move logs into FirebaseDB

Browse Source
This commit is contained in:
Michał Zieliński
2025-03-03 13:06:53 +01:00
parent f02ddd42fa
commit eb65ce6ae1
12 changed files with 257 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
WebAPI/Controllers/AdminController.cs
View File

@@ -1,4 +1,5 @@
using System.Data;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Data.SqlClient;
@@ -17,11 +18,12 @@ public class AdminController : Controller
public AdminController(
GoogleDriveHelper googleDriveHelper,
GoogleSheetsHelper googleSheetsHelper,
IConfiguration configuration)
IConfiguration configuration,
FirestoreDb firestoreDb)
{
_googleDriveHelper = googleDriveHelper;
_configuration = configuration;
_logsController = new LogsController(googleSheetsHelper, _configuration);
_logsController = new LogsController(firestoreDb);
}
[HttpGet]

6
WebAPI/Controllers/DataInboxController.cs
View File

@@ -1,5 +1,6 @@
using System.Data;
using System.Text;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Data.SqlClient;
@@ -19,11 +20,12 @@ public class DataInboxController : Controller
public DataInboxController(
AppDbContext db,
GoogleSheetsHelper googleSheetsHelper,
IConfiguration configuration)
IConfiguration configuration,
FirestoreDb firestoreDb)
{
_db = db;
_configuration = configuration;
_logsController = new LogsController(googleSheetsHelper, _configuration);
_logsController = new LogsController(firestoreDb);
}
[HttpPut]

345
WebAPI/Controllers/LayersController.cs
View File

@@ -2,6 +2,7 @@ using System.Globalization;
using System.Text;
using DiunaBIWebAPI.dataImporters;
using Google.Apis.Sheets.v4;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
@@ -25,7 +26,8 @@ public class LayersController : Controller
AppDbContext db,
GoogleSheetsHelper googleSheetsHelper,
GoogleDriveHelper googleDriveHelper,
IConfiguration configuration
IConfiguration configuration,
FirestoreDb firestoreDb
)
{
_db = db;
@@ -36,7 +38,7 @@ public class LayersController : Controller
_googleDriveHelper = googleDriveHelper;
_configuration = configuration;
_logsController = new LogsController(googleSheetsHelper, _configuration);
_logsController = new LogsController(firestoreDb);
}
[HttpGet]
@@ -330,42 +332,9 @@ public class LayersController : Controller
});
break;
case "FK2":
{
var fk2Importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
fk2Importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
break;
}
default:
{
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
if (startDate == null)
{
throw new Exception("StartDate record nod found");
}
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
if (endDate == null)
{
throw new Exception("EndDate record nod found");
}
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date &&
endDateParsed.Date >= DateTime.UtcNow.Date)
{
var importer = new MorskaImporter(_db, _googleSheetValues, this);
importer.Import(importWorker);
var fk2Importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
fk2Importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
@@ -376,36 +345,69 @@ public class LayersController : Controller
Message = "Success",
CreatedAt = DateTime.UtcNow
});
break;
}
else if (IsImportedLayerUpToDate(importWorker) == false)
default:
{
var importer = new MorskaImporter(_db, _googleSheetValues, this);
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1;
if (startDate == null)
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "Success (reimported)",
CreatedAt = DateTime.UtcNow
});
}
else
{
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "importLayer records are up to date. Not processed.",
CreatedAt = DateTime.UtcNow
});
}
throw new Exception("StartDate record nod found");
}
break;
}
var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1;
if (endDate == null)
{
throw new Exception("EndDate record nod found");
}
var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null);
var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null);
if (startDateParsed.Date <= DateTime.UtcNow.Date &&
endDateParsed.Date >= DateTime.UtcNow.Date)
{
var importer = new MorskaImporter(_db, _googleSheetValues, this);
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Import,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
}
else if (IsImportedLayerUpToDate(importWorker) == false)
{
var importer = new MorskaImporter(_db, _googleSheetValues, this);
importer.Import(importWorker);
Thread.Sleep(5000); // be aware of GSheet API quota
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "Success (reimported)",
CreatedAt = DateTime.UtcNow
});
}
else
{
_logsController.AddEntry(new LogEntry
{
Title = $"{importWorker.Name}, {importWorker.Id}",
Type = LogEntryType.Warning,
LogType = LogType.Import,
Message = "importLayer records are up to date. Not processed.",
CreatedAt = DateTime.UtcNow
});
}
break;
}
}
}
catch (Exception e)
@@ -483,6 +485,7 @@ public class LayersController : Controller
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type)
&& x.Number == 5586
)
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
@@ -553,98 +556,98 @@ public class LayersController : Controller
case null:
throw new Exception("ProcessType record not found");
case "T3-SourceYearSummary":
{
var processor =
new T3SourceYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor =
new T3SourceYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T3-MultiSourceYearSummary":
{
var processor =
new T3MultiSourceYearSummaryProcessor(_db, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor =
new T3MultiSourceYearSummaryProcessor(_db, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T3-MultiSourceCopySelectedCodesYearSummary":
{
var processor =
new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor =
new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T1-R1":
{
var processor = new T1R1Processor(_db, _googleSheetValues, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor = new T1R1Processor(_db, _googleSheetValues, this, _logsController);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T4-R2":
{
var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
case "T1-R3":
{
var processor = new T1R3Processor(_db, this, _googleSheetValues);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
var processor = new T1R3Processor(_db, this, _googleSheetValues);
processor.Process(processWorker);
_logsController.AddEntry(new LogEntry
{
Title = $"{processWorker.Name}, {processWorker.Id}",
Type = LogEntryType.Info,
LogType = LogType.Process,
Message = "Success",
CreatedAt = DateTime.UtcNow
});
return;
}
}
var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1;
@@ -656,37 +659,37 @@ public class LayersController : Controller
switch (processType!)
{
case "T3-SingleSource":
{
var t3SingleSource = new T3SingleSourceProcessor(_db, this, _logsController);
t3SingleSource.Process(processWorker);
break;
}
{
var t3SingleSource = new T3SingleSourceProcessor(_db, this);
t3SingleSource.Process(processWorker);
break;
}
case "T4-SingleSource":
{
var t4SingleSource = new T4SingleSourceProcessor(_db, this);
t4SingleSource.Process(processWorker);
break;
}
{
var t4SingleSource = new T4SingleSourceProcessor(_db, this);
t4SingleSource.Process(processWorker);
break;
}
case "T5-LastValues":
{
var t5LastValues = new T5LastValuesProcessor(_db, this);
t5LastValues.Process(processWorker);
break;
}
{
var t5LastValues = new T5LastValuesProcessor(_db, this);
t5LastValues.Process(processWorker);
break;
}
case "T3-MultiSourceSummary":
{
var t3MultiSourceSummary =
new T3MultiSourceSummaryProcessor(_db, this, _logsController);
t3MultiSourceSummary.Process(processWorker);
break;
}
{
var t3MultiSourceSummary =
new T3MultiSourceSummaryProcessor(_db, this, _logsController);
t3MultiSourceSummary.Process(processWorker);
break;
}
case "T3-MultiSourceCopySelectedCodes":
{
var t3MultiSourceCopySelectedCode =
new T3MultiSourceCopySelectedCodesProcessor(_db, this);
t3MultiSourceCopySelectedCode.Process(processWorker);
break;
}
{
var t3MultiSourceCopySelectedCode =
new T3MultiSourceCopySelectedCodesProcessor(_db, this);
t3MultiSourceCopySelectedCode.Process(processWorker);
break;
}
}
_logsController.AddEntry(new LogEntry

71
WebAPI/Controllers/LogsController.cs
View File

@@ -1,6 +1,7 @@
using System.Globalization;
using Google.Apis.Sheets.v4;
using Google.Apis.Sheets.v4.Data;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Mvc;
using WebAPI.Models;
@@ -8,59 +9,35 @@ namespace WebAPI.Controllers;
public class LogsController : Controller
{
private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues;
private readonly IConfiguration _configuration;
private readonly FirestoreDb _firestoreDb;
private readonly Guid _SessionId = Guid.NewGuid();
public LogsController(
GoogleSheetsHelper googleSheetsHelper,
IConfiguration configuration)
FirestoreDb firestoreDb
)
{
if (googleSheetsHelper.Service is not null) {
_googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values;
}
_configuration = configuration;
_firestoreDb = firestoreDb;
}
public void AddEntry(LogEntry entry)
{
if (_googleSheetValues is null) {
throw new Exception("Google Sheets API not initialized");
entry.SessionId = _SessionId;
entry.Instance = LogInstance.Morska;
try {
var collection = _firestoreDb.Collection("ApiLogs");
var document = collection.Document();
document.SetAsync(new {
entry.Message,
entry.Title,
Type = Enum.GetName(typeof(LogEntryType), entry.Type),
LogType = Enum.GetName(typeof(LogType), entry.LogType),
Instance = Enum.GetName(typeof(LogInstance), entry.Instance),
entry.CreatedAt,
SessionId = entry.SessionId.ToString()
}).Wait();
} catch (Exception e) {
Console.WriteLine(e.Message);
}
// until we move logs into firebase disable save for info and warnings
if (entry.Type == LogEntryType.Info || entry.Type == LogEntryType.Warning) {
return;
}
var type = entry.LogType switch
{
LogType.Import => "Import",
LogType.Backup => "Backup",
LogType.Process => "Process",
LogType.PowerBi => "PowerBIAccess",
LogType.DataInbox => "DataInbox",
LogType.Queue => "Queue",
_ => "Other"
};
var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();
var data = response.Values;
var row = 1;
if (data != null) {
row = data.Count + 1;
}
var range = $"{type}!A{row}:D{row}";
var logRow = new List<object>
{
entry.CreatedAt.ToString(new CultureInfo("pl-PL")),
entry.Type.ToString(),
entry.Title!,
entry.Message!
};
var valueRange = new ValueRange { Values = new IList<object>[] { logRow }};
var updateRequest = _googleSheetValues.Update(valueRange, _configuration["appLogsFile"], range);
updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
updateRequest.Execute();
}
}