Move logs into FirebaseDB

.gitignore

@@ -59,3 +59,4 @@ _ReSharper*/
 
 .vs/
 Temp/
+WebAPI/diunabi-admin-firebase-Development.json

WebAPI/Controllers/AdminController.cs

@@ -1,4 +1,5 @@
 using System.Data;
+using Google.Cloud.Firestore;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Data.SqlClient;
@@ -17,11 +18,12 @@ public class AdminController : Controller
     public AdminController(
         GoogleDriveHelper googleDriveHelper,
         GoogleSheetsHelper googleSheetsHelper,
-        IConfiguration configuration)
+        IConfiguration configuration,
+        FirestoreDb firestoreDb)
     {
         _googleDriveHelper = googleDriveHelper;
         _configuration = configuration;
-        _logsController = new LogsController(googleSheetsHelper, _configuration);
+        _logsController = new LogsController(firestoreDb);
     }
 
     [HttpGet]

WebAPI/Controllers/DataInboxController.cs

@@ -1,5 +1,6 @@
 using System.Data;
 using System.Text;
+using Google.Cloud.Firestore;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Data.SqlClient;
@@ -19,11 +20,12 @@ public class DataInboxController : Controller
     public DataInboxController(
         AppDbContext db,
         GoogleSheetsHelper googleSheetsHelper,
-        IConfiguration configuration)
+        IConfiguration configuration,
+        FirestoreDb firestoreDb)
     {
         _db = db;
         _configuration = configuration;
-        _logsController = new LogsController(googleSheetsHelper, _configuration);
+        _logsController = new LogsController(firestoreDb);
     }
 
     [HttpPut]

WebAPI/Controllers/LayersController.cs

@@ -2,6 +2,7 @@ using System.Globalization;
 using System.Text;
 using DiunaBIWebAPI.dataImporters;
 using Google.Apis.Sheets.v4;
+using Google.Cloud.Firestore;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
@@ -25,7 +26,8 @@ public class LayersController : Controller
         AppDbContext db,
         GoogleSheetsHelper googleSheetsHelper,
         GoogleDriveHelper googleDriveHelper,
-        IConfiguration configuration
+        IConfiguration configuration,
+        FirestoreDb firestoreDb
     )
     {
         _db = db;
@@ -36,7 +38,7 @@ public class LayersController : Controller
 
         _googleDriveHelper = googleDriveHelper;
         _configuration = configuration;
-        _logsController = new LogsController(googleSheetsHelper, _configuration);
+        _logsController = new LogsController(firestoreDb);
     }
 
     [HttpGet]
@@ -483,6 +485,7 @@ public class LayersController : Controller
                         x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
                         x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
                         x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == type)
+                        && x.Number == 5586
                     )
                     .OrderBy(x => x.CreatedAt)
                     .AsNoTracking()
@@ -657,7 +660,7 @@ public class LayersController : Controller
         {
             case "T3-SingleSource":
                 {
-                var t3SingleSource = new T3SingleSourceProcessor(_db, this, _logsController);
+                    var t3SingleSource = new T3SingleSourceProcessor(_db, this);
                     t3SingleSource.Process(processWorker);
                     break;
                 }

WebAPI/Controllers/LogsController.cs

@@ -1,6 +1,7 @@
 using System.Globalization;
 using Google.Apis.Sheets.v4;
 using Google.Apis.Sheets.v4.Data;
+using Google.Cloud.Firestore;
 using Microsoft.AspNetCore.Mvc;
 using WebAPI.Models;
 
@@ -8,59 +9,35 @@ namespace WebAPI.Controllers;
 
 public class LogsController : Controller
 {
-    private readonly SpreadsheetsResource.ValuesResource? _googleSheetValues;
+
-    private readonly IConfiguration _configuration;
+    private readonly FirestoreDb _firestoreDb;
+    private readonly Guid _SessionId = Guid.NewGuid();
     public LogsController(
-        GoogleSheetsHelper googleSheetsHelper,
+        FirestoreDb firestoreDb
-        IConfiguration configuration)
+    )
     {
-        if (googleSheetsHelper.Service is not null) { 
+        _firestoreDb = firestoreDb;
-            _googleSheetValues = googleSheetsHelper.Service.Spreadsheets.Values;
-        }
-        _configuration = configuration;
     }
 
     public void AddEntry(LogEntry entry)
     {
-        if (_googleSheetValues is null) {
+        entry.SessionId = _SessionId;
-            throw new Exception("Google Sheets API not initialized");
+        entry.Instance = LogInstance.Morska;
-        }
 
-        // until we move logs into firebase disable save for info and warnings
+        try {
-        if (entry.Type == LogEntryType.Info || entry.Type == LogEntryType.Warning) {
+            var collection = _firestoreDb.Collection("ApiLogs");
-            return;
+            var document = collection.Document();
-        }
+            document.SetAsync(new {
-
+                entry.Message,
-        var type = entry.LogType switch
+                entry.Title,
-        {
+                Type = Enum.GetName(typeof(LogEntryType), entry.Type),
-            LogType.Import => "Import",
+                LogType = Enum.GetName(typeof(LogType), entry.LogType),
-            LogType.Backup => "Backup",
+                Instance = Enum.GetName(typeof(LogInstance), entry.Instance),
-            LogType.Process => "Process",
+                entry.CreatedAt,
-            LogType.PowerBi => "PowerBIAccess",
+                SessionId = entry.SessionId.ToString()
-            LogType.DataInbox => "DataInbox",
+            }).Wait();
-            LogType.Queue => "Queue",
+        } catch (Exception e) {
-            _ => "Other"
+            Console.WriteLine(e.Message);
-        };
+        }
-        var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();
-        var data = response.Values;
-        var row = 1;
-        if (data != null) {
-            row = data.Count + 1;
-        }
-        var range = $"{type}!A{row}:D{row}";
-
-        var logRow = new List<object>
-        {
-            entry.CreatedAt.ToString(new CultureInfo("pl-PL")),
-            entry.Type.ToString(),
-            entry.Title!,
-            entry.Message!
-        };
-
-        var valueRange = new ValueRange { Values = new IList<object>[] { logRow }};
-
-        var updateRequest = _googleSheetValues.Update(valueRange, _configuration["appLogsFile"], range);
-        updateRequest.ValueInputOption = SpreadsheetsResource.ValuesResource.UpdateRequest.ValueInputOptionEnum.RAW;
-        updateRequest.Execute();
     }
 }

WebAPI/DiunaBI-WebAPI.csproj

@@ -10,9 +10,11 @@
   <ItemGroup>
     <PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
     <PackageReference Include="CsvHelper" Version="33.0.1" />
+    <PackageReference Include="FirebaseAdmin" Version="3.1.0" />
     <PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
     <PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3627" />
     <PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3624" />
+    <PackageReference Include="Google.Cloud.Firestore" Version="3.9.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="9.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.0" />

WebAPI/GoogleSheetsHelper.cs

@@ -24,7 +24,6 @@ public class GoogleSheetsHelper
     }
     private static GoogleCredential GetCredentialsFromFile()
     {
-        // ReSharper disable once RedundantAssignment
         var fileName = "client_secrets.json";
 #if DEBUG
         fileName = "client_secrets.Development.json";

WebAPI/Models/LogEntry.cs

@@ -1,3 +1,5 @@
+using Google.Cloud.Firestore;
+
 namespace WebAPI.Models;
 
 public enum LogEntryType
@@ -6,7 +8,8 @@ public enum LogEntryType
     Warning,
     Error
 }
-public enum LogType {
+public enum LogType
+{
     Import,
     Backup,
     Process,
@@ -14,6 +17,11 @@ public enum LogType {
     DataInbox,
     Queue
 }
+
+public enum LogInstance
+{
+    Morska
+}
 public class LogEntry
 {
     public LogType LogType { get; init; }
@@ -21,4 +29,6 @@ public class LogEntry
     public string? Message { get; init; }
     public string? Title { get; init; }
     public DateTime CreatedAt { get; init; }
+    public Guid SessionId { get; set; }
+    public LogInstance Instance { get; set; }
 }

WebAPI/Program.cs

@@ -1,3 +1,6 @@
+using FirebaseAdmin;
+using Google.Apis.Auth.OAuth2;
+using Google.Cloud.Firestore;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.IdentityModel.Tokens;
@@ -8,7 +11,8 @@ using WebAPI;
 var builder = WebApplication.CreateBuilder(args);
 
 var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
-builder.Services.AddDbContext<AppDbContext>(x => {
+builder.Services.AddDbContext<AppDbContext>(x =>
+{
     x.UseSqlServer(connectionString);
     x.EnableSensitiveDataLogging();
 });
@@ -54,6 +58,18 @@ builder.Services.AddAuthentication();
 builder.Services.AddSingleton(typeof(GoogleSheetsHelper));
 builder.Services.AddSingleton(typeof(GoogleDriveHelper));
 
+var fileName = "diunabi-admin-firebase.json";
+#if DEBUG
+fileName = "diunabi-admin-firebase-Development.json";
+#endif
+var credentialPath = Path.Combine(Directory.GetCurrentDirectory(), fileName);
+System.Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", credentialPath);
+FirebaseAdmin.FirebaseApp.Create(new AppOptions()
+{
+    Credential = GoogleCredential.GetApplicationDefault()
+});
+builder.Services.AddSingleton(FirestoreDb.Create("diunabi-admin"));
+
 var app = builder.Build();
 
 
@@ -62,7 +78,8 @@ app.Use(async (context, next) =>
     var token = context.Request.Headers.Authorization.ToString();
     if (token.Length > 0
         && !context.Request.Path.ToString().Contains("getForPowerBI")
-        && !context.Request.Path.ToString().Contains("DataInbox/Add")) {
+        && !context.Request.Path.ToString().Contains("DataInbox/Add"))
+    {
         var handler = new JwtSecurityTokenHandler();
         var data = handler.ReadJwtToken(token.Split(' ')[1]);
         context.Request.Headers.Append("UserId", new Microsoft.Extensions.Primitives.StringValues(data.Subject));

WebAPI/dataProcessors/t3.SingleSource.processor.cs

@@ -7,8 +7,7 @@ namespace WebAPI.dataProcessors;
 
 public class T3SingleSourceProcessor(
     AppDbContext db,
-    LayersController controller,
+    LayersController controller)
-        LogsController logsController)
 {
     public void Process(Layer processWorker)
     {

WebAPI/diunabi-admin-firebase.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "#{firebase-project-id}#",
+  "private_key_id": "#{firebase-private-key-id}#",
+  "private_key": "#{firebase-private-key}#",
+  "client_email": "#{firebase-client-email}#",
+  "client_id": "#{firebase-client-id}#",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "#{firebase-client-cert-url}#",
+  "universe_domain": "googleapis.com"
+}