mz/DiunaBI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Final fixes before tests

Browse Source
This commit is contained in:
Michał Zieliński
2025-06-08 10:18:52 +02:00
parent 117a67ab58
commit ff41a71484
10 changed files with 374 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
src/Backend/DiunaBI.WebAPI/Controllers/LayersController.cs
View File

@@ -676,147 +676,19 @@ public class LayersController : Controller
}
}
}
[HttpGet]
[Route("GetImportWorkers")]
[AllowAnonymous]
public async Task<IActionResult> GetImportWorkers()
{
try
{
var importWorkerLayers = await _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
&& x.Number == 5579
)
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("GetImportWorkers: Found {LayerCount} import worker layers to queue",
importWorkerLayers.Count);
int queuedCount = 0;
foreach (var importWorker in importWorkerLayers)
{
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("GetImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
importWorker.Name, importWorker.Id);
continue;
}
// Check if plugin exists
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, importWorker.Name, importWorker.Id);
continue;
}
var job = new QueueJob
{
LayerId = importWorker.Id,
LayerName = importWorker.Name ?? "Unknown",
PluginName = pluginName,
JobType = JobType.Import,
Priority = 0, // All imports have same priority
MaxRetries = 5,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("GetImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("GetImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new {
Message = $"Queued {queuedCount} import jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = importWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "GetImportWorkers: Error queuing import workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("CheckProcessors")]
[AllowAnonymous]
public IActionResult CheckProcessors()
{
// get list od all enabled processors and check if they has record 'Plugin'
var enabledProcessors = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
// get
_logger.LogInformation("CheckProcessors: Found {ProcessorCount} enabled processors", enabledProcessors.Count);
foreach (var processor in enabledProcessors)
{
var pluginRecord = processor.Records!.FirstOrDefault(x => x.Code == "Plugin");
if (pluginRecord == null)
{
_logger.LogWarning("CheckProcessors: No Plugin record found for processor {ProcessorName} ({ProcessorId}), skipping",
processor.Name, processor.Id);
continue;
}
var pluginName = pluginRecord.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("CheckProcessors: Empty Plugin name for processor {ProcessorName} ({ProcessorId}), skipping",
processor.Name, processor.Id);
continue;
}
var processorInstance = _pluginManager.GetProcessor(pluginName);
if (processorInstance == null)
{
_logger.LogWarning("CheckProcessors: Processor {PluginName} not found for {ProcessorName} ({ProcessorId}), skipping",
pluginName, processor.Name, processor.Id);
continue;
}
}
_logger.LogInformation("CheckProcessors: Completed checking processors");
return Ok();
}
[HttpGet]
[Route("AddPluginName")]
[AllowAnonymous]
public IActionResult AddPluginName()
{
/*
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
x.Records!.Any(y => y.Code == "ProcessType" && y.Desc1 == "T1-R3")
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
@@ -829,20 +701,21 @@ public class LayersController : Controller
Id = Guid.NewGuid(),
LayerId = importWorker.Id,
Code = "Plugin",
Desc1 = "Morska.Process.T1.R3",
Desc1 = "Morska.Import.D3",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
_db.Records.Add(record);
//_db.Records.Add(record);
}
_db.SaveChanges();
//_db.SaveChanges();
*/
return Ok();
}
private static void WriteToConsole(params string[] messages)
{
foreach (var message in messages)
@@ -936,4 +809,277 @@ public class LayersController : Controller
throw;
}
}
[HttpGet]
[Route("EnqueueImportWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var importWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}",
importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (importWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No import workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var importWorker in importWorkerLayers)
{
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
importWorker.Name, importWorker.Id);
continue;
}
// Check if plugin exists
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, importWorker.Name, importWorker.Id);
continue;
}
var job = new QueueJob
{
LayerId = importWorker.Id,
LayerName = importWorker.Name ?? "Unknown",
PluginName = pluginName,
JobType = JobType.Import,
Priority = 0, // All imports have same priority
MaxRetries = 5,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} import jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = importWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("EnqueueProcessWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var processWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}",
processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (processWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No process workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var processWorker in processWorkerLayers)
{
var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
processWorker.Name, processWorker.Id);
continue;
}
var processorType = processWorker.Records!.FirstOrDefault(x => x.Code == "ProcessorType")?.Desc1;
if (string.IsNullOrEmpty(processorType))
{
_logger.LogWarning("EnqueueProcessWorkers: No processor type found for layer {LayerName} ({LayerId}), skipping",
processWorker.Name, processWorker.Id);
continue;
}
// Check if processor exists
var processor = _pluginManager.GetProcessor(processorType);
if (processor == null)
{
_logger.LogWarning("EnqueueProcessWorkers: Processor {ProcessorType} not found for layer {LayerName} ({LayerId}), skipping",
processorType, processWorker.Name, processWorker.Id);
continue;
}
// Get priority from ProcessWorker record, default to 10 if not found
var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1;
var priority = 10; // Default priority
if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority))
{
priority = parsedPriority;
}
var job = new QueueJob
{
LayerId = processWorker.Id,
LayerName = processWorker.Name ?? "Unknown",
PluginName = processorType, // Use processorType as PluginName for process jobs
JobType = JobType.Process,
Priority = priority,
MaxRetries = 3,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with processor {ProcessorType}, priority {Priority}",
processWorker.Name, processWorker.Id, processorType, priority);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} process jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = processWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("RunQueueJobs/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> RunQueueJobs(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var queueSize = await _queueService.GetQueueCountAsync();
if (queueSize == 0)
{
return Ok(new
{
Message = "Queue is empty",
QueueSize = 0,
Status = "No jobs to process"
});
}
_logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize);
// PRZYWRÓĆ SINGLETON ACCESS:
var queueProcessor = HttpContext.RequestServices.GetRequiredService<JobQueueProcessor>();
queueProcessor.TriggerProcessing();
return Ok(new
{
Message = $"Queue processing triggered for {queueSize} jobs",
QueueSize = queueSize,
Status = "Processing started in background"
});
}
catch (Exception e)
{
_logger.LogError(e, "RunQueueJobs: Error triggering queue processing");
return BadRequest(e.ToString());
}
}
}

23
src/Backend/DiunaBI.WebAPI/Program.cs
View File

@@ -1,5 +1,3 @@
using Google.Apis.Auth.OAuth2;
using Google.Cloud.Firestore;
using Microsoft.AspNetCore.Authentication.JwtBearer;
using Microsoft.EntityFrameworkCore;
using Microsoft.IdentityModel.Tokens;
@@ -14,7 +12,6 @@ using DiunaBI.Core.Interfaces;
var builder = WebApplication.CreateBuilder(args);
// ✅ SERILOG TYLKO DLA PRODUKCJI
if (builder.Environment.IsProduction())
{
builder.Host.UseSerilog((context, configuration) =>
@@ -30,6 +27,7 @@ if (builder.Environment.IsProduction())
}
var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
builder.Services.AddDbContext<AppDbContext>(x =>
{
x.UseSqlServer(connectionString);
@@ -70,25 +68,24 @@ builder.Services.AddAuthentication(options =>
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!))
};
});
builder.Services.AddAuthentication();
// Queue services
builder.Services.AddScoped<IJobQueueService, JobQueueService>();
builder.Services.AddHostedService<JobQueueProcessor>();
builder.Services.AddSingleton<JobQueueProcessor>();
// Zarejestruj Google Sheets dependencies
// Google Sheets dependencies
builder.Services.AddSingleton<GoogleSheetsHelper>();
builder.Services.AddSingleton<GoogleDriveHelper>();
builder.Services.AddSingleton<SpreadsheetsResource.ValuesResource>(provider =>
{
var googleSheetsHelper = provider.GetRequiredService<GoogleSheetsHelper>();
var valuesResource = googleSheetsHelper.Service?.Spreadsheets.Values;
if (valuesResource == null)
{
throw new InvalidOperationException("Google Sheets Service is not initialized properly");
}
return valuesResource;
});
@@ -96,7 +93,6 @@ builder.Services.AddSingleton<PluginManager>();
var app = builder.Build();
// ✅ SERILOG REQUEST LOGGING TYLKO DLA PRODUKCJI
if (app.Environment.IsProduction())
{
app.UseSerilogRequestLogging(options =>
@@ -106,25 +102,25 @@ if (app.Environment.IsProduction())
{
diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value);
diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme);
var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault();
if (!string.IsNullOrEmpty(userAgent))
{
diagnosticContext.Set("UserAgent", userAgent);
}
diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown");
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
};
});
}
// Plugin initialization
var pluginManager = app.Services.GetRequiredService<PluginManager>();
var executablePath = Assembly.GetExecutingAssembly().Location;
var executableDir = Path.GetDirectoryName(executablePath)!;
var pluginsPath = Path.Combine(executableDir, "Plugins");
// ✅ RÓŻNE LOGGERY W ZALEŻNOŚCI OD ŚRODOWISKA
if (app.Environment.IsProduction())
{
Log.Information("Starting DiunaBI application");
@@ -165,8 +161,7 @@ app.MapControllers();
app.Run();
// ✅ SERILOG CLEANUP TYLKO DLA PRODUKCJI
if (app.Environment.IsProduction())
{
Log.CloseAndFlush();
}
}