diff --git a/.gitignore b/.gitignore index ca8d8f1..738761e 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,5 @@ _ReSharper*/ *.DotSettings.user .vs/ -Temp/ \ No newline at end of file +Temp/ +WebAPI/diunabi-admin-firebase-Development.json diff --git a/WebAPI/.DS_Store b/WebAPI/.DS_Store index c9ea3a6..e8ba1c4 100644 Binary files a/WebAPI/.DS_Store and b/WebAPI/.DS_Store differ diff --git a/WebAPI/Controllers/AdminController.cs b/WebAPI/Controllers/AdminController.cs index ff16fb8..8f89d20 100644 --- a/WebAPI/Controllers/AdminController.cs +++ b/WebAPI/Controllers/AdminController.cs @@ -1,4 +1,5 @@ using System.Data; +using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Data.SqlClient; @@ -17,11 +18,12 @@ public class AdminController : Controller public AdminController( GoogleDriveHelper googleDriveHelper, GoogleSheetsHelper googleSheetsHelper, - IConfiguration configuration) + IConfiguration configuration, + FirestoreDb firestoreDb) { _googleDriveHelper = googleDriveHelper; _configuration = configuration; - _logsController = new LogsController(googleSheetsHelper, _configuration); + _logsController = new LogsController(firestoreDb); } [HttpGet] diff --git a/WebAPI/Controllers/DataInboxController.cs b/WebAPI/Controllers/DataInboxController.cs index e886d1e..ca2b618 100644 --- a/WebAPI/Controllers/DataInboxController.cs +++ b/WebAPI/Controllers/DataInboxController.cs @@ -1,5 +1,6 @@ using System.Data; using System.Text; +using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Data.SqlClient; @@ -19,11 +20,12 @@ public class DataInboxController : Controller public DataInboxController( AppDbContext db, GoogleSheetsHelper googleSheetsHelper, - IConfiguration configuration) + IConfiguration configuration, + FirestoreDb firestoreDb) { _db = db; _configuration = configuration; - _logsController = new LogsController(googleSheetsHelper, _configuration); + _logsController = new LogsController(firestoreDb); } [HttpPut] diff --git a/WebAPI/Controllers/LayersController.cs b/WebAPI/Controllers/LayersController.cs index a2f1a68..2712aa3 100644 --- a/WebAPI/Controllers/LayersController.cs +++ b/WebAPI/Controllers/LayersController.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Text; using DiunaBIWebAPI.dataImporters; using Google.Apis.Sheets.v4; +using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; @@ -25,7 +26,8 @@ public class LayersController : Controller AppDbContext db, GoogleSheetsHelper googleSheetsHelper, GoogleDriveHelper googleDriveHelper, - IConfiguration configuration + IConfiguration configuration, + FirestoreDb firestoreDb ) { _db = db; @@ -36,7 +38,7 @@ public class LayersController : Controller _googleDriveHelper = googleDriveHelper; _configuration = configuration; - _logsController = new LogsController(googleSheetsHelper, _configuration); + _logsController = new LogsController(firestoreDb); } [HttpGet] @@ -330,42 +332,9 @@ public class LayersController : Controller }); break; case "FK2": - { - var fk2Importer = new MorskaFk2Importer(_db, _googleSheetValues, this); - fk2Importer.Import(importWorker); - Thread.Sleep(5000); // be aware of GSheet API quota - - _logsController.AddEntry(new LogEntry { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Import, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - break; - } - default: - { - var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; - if (startDate == null) - { - throw new Exception("StartDate record nod found"); - } - - var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; - if (endDate == null) - { - throw new Exception("EndDate record nod found"); - } - - var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); - var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); - if (startDateParsed.Date <= DateTime.UtcNow.Date && - endDateParsed.Date >= DateTime.UtcNow.Date) - { - var importer = new MorskaImporter(_db, _googleSheetValues, this); - importer.Import(importWorker); + var fk2Importer = new MorskaFk2Importer(_db, _googleSheetValues, this); + fk2Importer.Import(importWorker); Thread.Sleep(5000); // be aware of GSheet API quota _logsController.AddEntry(new LogEntry @@ -376,36 +345,69 @@ public class LayersController : Controller Message = "Success", CreatedAt = DateTime.UtcNow }); + break; } - else if (IsImportedLayerUpToDate(importWorker) == false) + default: { - var importer = new MorskaImporter(_db, _googleSheetValues, this); - importer.Import(importWorker); - Thread.Sleep(5000); // be aware of GSheet API quota - - _logsController.AddEntry(new LogEntry + var startDate = importWorker.Records!.FirstOrDefault(x => x.Code == "StartDate")?.Desc1; + if (startDate == null) { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Import, - Message = "Success (reimported)", - CreatedAt = DateTime.UtcNow - }); - } - else - { - _logsController.AddEntry(new LogEntry - { - Title = $"{importWorker.Name}, {importWorker.Id}", - Type = LogEntryType.Warning, - LogType = LogType.Import, - Message = "importLayer records are up to date. Not processed.", - CreatedAt = DateTime.UtcNow - }); - } + throw new Exception("StartDate record nod found"); + } - break; - } + var endDate = importWorker.Records!.First(x => x.Code == "EndDate").Desc1; + if (endDate == null) + { + throw new Exception("EndDate record nod found"); + } + + var startDateParsed = DateTime.ParseExact(startDate, "yyyy.MM.dd", null); + var endDateParsed = DateTime.ParseExact(endDate, "yyyy.MM.dd", null); + if (startDateParsed.Date <= DateTime.UtcNow.Date && + endDateParsed.Date >= DateTime.UtcNow.Date) + { + var importer = new MorskaImporter(_db, _googleSheetValues, this); + importer.Import(importWorker); + Thread.Sleep(5000); // be aware of GSheet API quota + + _logsController.AddEntry(new LogEntry + { + Title = $"{importWorker.Name}, {importWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Import, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + } + else if (IsImportedLayerUpToDate(importWorker) == false) + { + var importer = new MorskaImporter(_db, _googleSheetValues, this); + importer.Import(importWorker); + Thread.Sleep(5000); // be aware of GSheet API quota + + _logsController.AddEntry(new LogEntry + { + Title = $"{importWorker.Name}, {importWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Import, + Message = "Success (reimported)", + CreatedAt = DateTime.UtcNow + }); + } + else + { + _logsController.AddEntry(new LogEntry + { + Title = $"{importWorker.Name}, {importWorker.Id}", + Type = LogEntryType.Warning, + LogType = LogType.Import, + Message = "importLayer records are up to date. Not processed.", + CreatedAt = DateTime.UtcNow + }); + } + + break; + } } } catch (Exception e) @@ -483,6 +485,7 @@ public class LayersController : Controller x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") && x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") && x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type) + && x.Number == 5586 ) .OrderBy(x => x.CreatedAt) .AsNoTracking() @@ -553,98 +556,98 @@ public class LayersController : Controller case null: throw new Exception("ProcessType record not found"); case "T3-SourceYearSummary": - { - var processor = - new T3SourceYearSummaryProcessor(_db, this); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = + new T3SourceYearSummaryProcessor(_db, this); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } case "T3-MultiSourceYearSummary": - { - var processor = - new T3MultiSourceYearSummaryProcessor(_db, this, _logsController); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = + new T3MultiSourceYearSummaryProcessor(_db, this, _logsController); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } case "T3-MultiSourceCopySelectedCodesYearSummary": - { - var processor = - new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = + new T3MultiSourceCopySelectedCodesYearSummaryProcessor(_db, this); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } case "T1-R1": - { - var processor = new T1R1Processor(_db, _googleSheetValues, this, _logsController); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = new T1R1Processor(_db, _googleSheetValues, this, _logsController); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } case "T4-R2": - { - var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = new T4R2Processor(_db, this, _logsController, _googleSheetValues); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } case "T1-R3": - { - var processor = new T1R3Processor(_db, this, _googleSheetValues); - processor.Process(processWorker); - - _logsController.AddEntry(new LogEntry { - Title = $"{processWorker.Name}, {processWorker.Id}", - Type = LogEntryType.Info, - LogType = LogType.Process, - Message = "Success", - CreatedAt = DateTime.UtcNow - }); - return; - } + var processor = new T1R3Processor(_db, this, _googleSheetValues); + processor.Process(processWorker); + + _logsController.AddEntry(new LogEntry + { + Title = $"{processWorker.Name}, {processWorker.Id}", + Type = LogEntryType.Info, + LogType = LogType.Process, + Message = "Success", + CreatedAt = DateTime.UtcNow + }); + return; + } } var month = processWorker.Records?.SingleOrDefault(x => x.Code == "Month")?.Desc1; @@ -656,37 +659,37 @@ public class LayersController : Controller switch (processType!) { case "T3-SingleSource": - { - var t3SingleSource = new T3SingleSourceProcessor(_db, this, _logsController); - t3SingleSource.Process(processWorker); - break; - } + { + var t3SingleSource = new T3SingleSourceProcessor(_db, this); + t3SingleSource.Process(processWorker); + break; + } case "T4-SingleSource": - { - var t4SingleSource = new T4SingleSourceProcessor(_db, this); - t4SingleSource.Process(processWorker); - break; - } + { + var t4SingleSource = new T4SingleSourceProcessor(_db, this); + t4SingleSource.Process(processWorker); + break; + } case "T5-LastValues": - { - var t5LastValues = new T5LastValuesProcessor(_db, this); - t5LastValues.Process(processWorker); - break; - } + { + var t5LastValues = new T5LastValuesProcessor(_db, this); + t5LastValues.Process(processWorker); + break; + } case "T3-MultiSourceSummary": - { - var t3MultiSourceSummary = - new T3MultiSourceSummaryProcessor(_db, this, _logsController); - t3MultiSourceSummary.Process(processWorker); - break; - } + { + var t3MultiSourceSummary = + new T3MultiSourceSummaryProcessor(_db, this, _logsController); + t3MultiSourceSummary.Process(processWorker); + break; + } case "T3-MultiSourceCopySelectedCodes": - { - var t3MultiSourceCopySelectedCode = - new T3MultiSourceCopySelectedCodesProcessor(_db, this); - t3MultiSourceCopySelectedCode.Process(processWorker); - break; - } + { + var t3MultiSourceCopySelectedCode = + new T3MultiSourceCopySelectedCodesProcessor(_db, this); + t3MultiSourceCopySelectedCode.Process(processWorker); + break; + } } _logsController.AddEntry(new LogEntry diff --git a/WebAPI/Controllers/LogsController.cs b/WebAPI/Controllers/LogsController.cs index c21df3d..6c93bcf 100644 --- a/WebAPI/Controllers/LogsController.cs +++ b/WebAPI/Controllers/LogsController.cs @@ -1,6 +1,7 @@ using System.Globalization; using Google.Apis.Sheets.v4; using Google.Apis.Sheets.v4.Data; +using Google.Cloud.Firestore; using Microsoft.AspNetCore.Mvc; using WebAPI.Models; @@ -8,59 +9,35 @@ namespace WebAPI.Controllers; public class LogsController : Controller { - private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues; - private readonly IConfiguration _configuration; + + private readonly FirestoreDb _firestoreDb; + private readonly Guid _SessionId = Guid.NewGuid(); public LogsController( - GoogleSheetsHelper googleSheetsHelper, - IConfiguration configuration) + FirestoreDb firestoreDb + ) { - if (googleSheetsHelper.Service is not null) { - _googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values; - } - _configuration = configuration; + _firestoreDb = firestoreDb; } public void AddEntry(LogEntry entry) { - if (_googleSheetValues is null) { - throw new Exception("Google Sheets API not initialized"); + entry.SessionId = _SessionId; + entry.Instance = LogInstance.Morska; + + try { + var collection = _firestoreDb.Collection("ApiLogs"); + var document = collection.Document(); + document.SetAsync(new { + entry.Message, + entry.Title, + Type = Enum.GetName(typeof(LogEntryType), entry.Type), + LogType = Enum.GetName(typeof(LogType), entry.LogType), + Instance = Enum.GetName(typeof(LogInstance), entry.Instance), + entry.CreatedAt, + SessionId = entry.SessionId.ToString() + }).Wait(); + } catch (Exception e) { + Console.WriteLine(e.Message); } - - // until we move logs into firebase disable save for info and warnings - if (entry.Type == LogEntryType.Info || entry.Type == LogEntryType.Warning) { - return; - } - - var type = entry.LogType switch - { - LogType.Import => "Import", - LogType.Backup => "Backup", - LogType.Process => "Process", - LogType.PowerBi => "PowerBIAccess", - LogType.DataInbox => "DataInbox", - LogType.Queue => "Queue", - _ => "Other" - }; - var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute(); - var data = response.Values; - var row = 1; - if (data != null) { - row = data.Count + 1; - } - var range = $"{type}!A{row}:D{row}"; - - var logRow = new List - { - entry.CreatedAt.ToString(new CultureInfo("pl-PL")), - entry.Type.ToString(), - entry.Title!, - entry.Message! - }; - - var valueRange = new ValueRange { Values = new IList[] { logRow }}; - - var updateRequest = _googleSheetValues.Update(valueRange, _configuration["appLogsFile"], range); - updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW; - updateRequest.Execute(); } } \ No newline at end of file diff --git a/WebAPI/DiunaBI-WebAPI.csproj b/WebAPI/DiunaBI-WebAPI.csproj index b227ba1..8ab073d 100644 --- a/WebAPI/DiunaBI-WebAPI.csproj +++ b/WebAPI/DiunaBI-WebAPI.csproj @@ -10,9 +10,11 @@ + + diff --git a/WebAPI/GoogleSheetsHelper.cs b/WebAPI/GoogleSheetsHelper.cs index 4a06596..a7dbecd 100644 --- a/WebAPI/GoogleSheetsHelper.cs +++ b/WebAPI/GoogleSheetsHelper.cs @@ -24,7 +24,6 @@ public class GoogleSheetsHelper } private static GoogleCredential GetCredentialsFromFile() { - // ReSharper disable once RedundantAssignment var fileName = "client_secrets.json"; #if DEBUG fileName = "client_secrets.Development.json"; diff --git a/WebAPI/Models/LogEntry.cs b/WebAPI/Models/LogEntry.cs index 4dccaf0..6d32e77 100644 --- a/WebAPI/Models/LogEntry.cs +++ b/WebAPI/Models/LogEntry.cs @@ -1,3 +1,5 @@ +using Google.Cloud.Firestore; + namespace WebAPI.Models; public enum LogEntryType @@ -6,7 +8,8 @@ public enum LogEntryType Warning, Error } -public enum LogType { +public enum LogType +{ Import, Backup, Process, @@ -14,11 +17,18 @@ public enum LogType { DataInbox, Queue } + +public enum LogInstance +{ + Morska +} public class LogEntry { public LogType LogType { get; init; } public LogEntryType Type { get; init; } public string? Message { get; init; } - public string? Title {get; init;} + public string? Title { get; init; } public DateTime CreatedAt { get; init; } + public Guid SessionId { get; set; } + public LogInstance Instance { get; set; } } \ No newline at end of file diff --git a/WebAPI/Program.cs b/WebAPI/Program.cs index eeb1d41..f4dfc13 100644 --- a/WebAPI/Program.cs +++ b/WebAPI/Program.cs @@ -1,3 +1,6 @@ +using FirebaseAdmin; +using Google.Apis.Auth.OAuth2; +using Google.Cloud.Firestore; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.EntityFrameworkCore; using Microsoft.IdentityModel.Tokens; @@ -8,10 +11,11 @@ using WebAPI; var builder = WebApplication.CreateBuilder(args); var connectionString = builder.Configuration.GetConnectionString("SQLDatabase"); -builder.Services.AddDbContext(x => { +builder.Services.AddDbContext(x => +{ x.UseSqlServer(connectionString); x.EnableSensitiveDataLogging(); - }); +}); builder.Services.AddCors(options => { @@ -54,15 +58,28 @@ builder.Services.AddAuthentication(); builder.Services.AddSingleton(typeof(GoogleSheetsHelper)); builder.Services.AddSingleton(typeof(GoogleDriveHelper)); +var fileName = "diunabi-admin-firebase.json"; +#if DEBUG +fileName = "diunabi-admin-firebase-Development.json"; +#endif +var credentialPath = Path.Combine(Directory.GetCurrentDirectory(), fileName); +System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialPath); +FirebaseAdmin.FirebaseApp.Create(new AppOptions() +{ + Credential = GoogleCredential.GetApplicationDefault() +}); +builder.Services.AddSingleton(FirestoreDb.Create("diunabi-admin")); + var app = builder.Build(); app.Use(async (context, next) => { var token = context.Request.Headers.Authorization.ToString(); - if (token.Length > 0 + if (token.Length > 0 && !context.Request.Path.ToString().Contains("getForPowerBI") - && !context.Request.Path.ToString().Contains("DataInbox/Add")) { + && !context.Request.Path.ToString().Contains("DataInbox/Add")) + { var handler = new JwtSecurityTokenHandler(); var data = handler.ReadJwtToken(token.Split(' ')[1]); context.Request.Headers.Append("UserId", new Microsoft.Extensions.Primitives.StringValues(data.Subject)); diff --git a/WebAPI/dataProcessors/t3.SingleSource.processor.cs b/WebAPI/dataProcessors/t3.SingleSource.processor.cs index e8e9ef6..7528b24 100644 --- a/WebAPI/dataProcessors/t3.SingleSource.processor.cs +++ b/WebAPI/dataProcessors/t3.SingleSource.processor.cs @@ -7,8 +7,7 @@ namespace WebAPI.dataProcessors; public class T3SingleSourceProcessor( AppDbContext db, - LayersController controller, - LogsController logsController) + LayersController controller) { public void Process(Layer processWorker) { diff --git a/WebAPI/diunabi-admin-firebase.json b/WebAPI/diunabi-admin-firebase.json new file mode 100644 index 0000000..a27955f --- /dev/null +++ b/WebAPI/diunabi-admin-firebase.json @@ -0,0 +1,13 @@ +{ + "type": "service_account", + "project_id": "#{firebase-project-id}#", + "private_key_id": "#{firebase-private-key-id}#", + "private_key": "#{firebase-private-key}#", + "client_email": "#{firebase-client-email}#", + "client_id": "#{firebase-client-id}#", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "#{firebase-client-cert-url}#", + "universe_domain": "googleapis.com" +}