Remove queue

This commit is contained in:
Michał Zieliński
2025-06-08 11:08:48 +02:00
parent 99d8593c49
commit 4fd0b1cd50
5 changed files with 2 additions and 777 deletions

View File

@@ -1,18 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using DiunaBI.Core.Models;
namespace DiunaBI.Core.Interfaces;
public interface IJobQueueService
{
Task EnqueueJobAsync(QueueJob job);
Task<QueueJob?> DequeueJobAsync(JobType? jobType = null);
Task MarkJobCompletedAsync(Guid jobId);
Task MarkJobFailedAsync(Guid jobId, string error);
Task MarkJobForRetryAsync(Guid jobId, string error);
Task<IEnumerable<QueueJob>> GetQueueStatusAsync();
Task<int> GetQueueCountAsync(JobType? jobType = null);
Task<int> GetRunningJobsCountAsync(JobType jobType);
}

View File

@@ -1,238 +0,0 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.EntityFrameworkCore;
using DiunaBI.Core.Models;
using DiunaBI.Core.Interfaces;
using DiunaBI.Core.Database.Context;
using System;
using System.Threading.Tasks;
using System.Threading;
using System.Net.Http;
using System.Linq;
namespace DiunaBI.Core.Services;
public class JobQueueProcessor : BackgroundService
{
private readonly IServiceScopeFactory _scopeFactory; // ✅ GOOD - używa scope factory
private readonly ILogger<JobQueueProcessor> _logger;
private readonly ManualResetEventSlim _processSignal = new(false);
// ❌ USUŃ DIRECT INJECTION scoped services:
// private readonly IJobQueueService _queueService;
// private readonly AppDbContext _db;
public JobQueueProcessor(
IServiceScopeFactory scopeFactory, // ✅ GOOD - inject scope factory
ILogger<JobQueueProcessor> logger)
{
_scopeFactory = scopeFactory;
_logger = logger;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("JobQueueProcessor: Started (manual trigger mode)");
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Wait for manual trigger
_processSignal.Wait(stoppingToken);
_processSignal.Reset();
_logger.LogInformation("JobQueueProcessor: Processing triggered");
// ✅ GOOD - create scope for each processing cycle
using var scope = _scopeFactory.CreateScope();
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
var pluginManager = scope.ServiceProvider.GetRequiredService<PluginManager>();
await ProcessQueueAsync(queueService, pluginManager, stoppingToken);
}
catch (OperationCanceledException)
{
_logger.LogInformation("JobQueueProcessor: Cancellation requested");
break;
}
catch (Exception ex)
{
_logger.LogError(ex, "JobQueueProcessor: Error in processing loop");
await Task.Delay(5000, stoppingToken); // Wait before retry
}
}
_logger.LogInformation("JobQueueProcessor: Stopped");
}
public void TriggerProcessing()
{
_logger.LogInformation("JobQueueProcessor: Manual trigger received");
_processSignal.Set();
}
private async Task ProcessQueueAsync(IJobQueueService queueService, PluginManager pluginManager, CancellationToken cancellationToken)
{
var startTime = DateTime.UtcNow;
var initialQueueSize = await queueService.GetQueueCountAsync();
int processedJobs = 0;
int failedJobs = 0;
_logger.LogInformation("JobQueueProcessor: Starting processing of {InitialQueueSize} jobs", initialQueueSize);
while (!cancellationToken.IsCancellationRequested)
{
// First process all imports (they run sequentially due to Google Sheets API limits)
var importJob = await queueService.DequeueJobAsync(JobType.Import);
if (importJob != null)
{
await ProcessJobAsync(importJob, cancellationToken);
if (importJob.Status == JobStatus.Completed) processedJobs++;
else failedJobs++;
// Add delay between imports to respect Google Sheets API limits
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
continue;
}
// Then process processors (can run in parallel within same priority)
var processJob = await queueService.DequeueJobAsync(JobType.Process);
if (processJob != null)
{
await ProcessJobAsync(processJob, cancellationToken);
if (processJob.Status == JobStatus.Completed) processedJobs++;
else failedJobs++;
continue;
}
// No more jobs in queue
break;
}
var endTime = DateTime.UtcNow;
var duration = endTime - startTime;
var finalQueueSize = await queueService.GetQueueCountAsync();
_logger.LogInformation("JobQueueProcessor: Processing completed. Duration: {Duration:hh\\:mm\\:ss}, " +
"Initial queue: {InitialQueueSize}, Processed: {ProcessedJobs}, Failed: {FailedJobs}, " +
"Final queue size: {FinalQueueSize}",
duration, initialQueueSize, processedJobs, failedJobs, finalQueueSize);
if (failedJobs > 0)
{
_logger.LogWarning("JobQueueProcessor: {FailedJobs} jobs failed during processing. Check logs for details.", failedJobs);
}
}
private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken)
{
try
{
_logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})",
job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority);
// POPRAWKA: używaj _scopeFactory zamiast _serviceProvider
using var scope = _scopeFactory.CreateScope();
var dbContext = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var pluginManager = scope.ServiceProvider.GetRequiredService<PluginManager>();
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
// Get the layer with records
var layer = await dbContext.Layers
.Include(x => x.Records)
.FirstOrDefaultAsync(x => x.Id == job.LayerId && !x.IsDeleted, cancellationToken);
if (layer == null)
{
_logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId);
await queueService.MarkJobFailedAsync(job.Id, "Layer not found");
job.Status = JobStatus.Failed;
return;
}
// Process based on job type
switch (job.JobType)
{
case JobType.Import:
var importer = pluginManager.GetImporter(job.PluginName);
if (importer == null)
{
_logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName);
await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found");
job.Status = JobStatus.Failed;
return;
}
_logger.LogInformation("JobQueueProcessor: Executing import for layer {LayerName} with plugin {PluginName}",
layer.Name, job.PluginName);
importer.Import(layer);
break;
case JobType.Process:
var processor = pluginManager.GetProcessor(job.PluginName);
if (processor == null)
{
_logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName);
await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found");
job.Status = JobStatus.Failed;
return;
}
_logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with processor {PluginName}",
layer.Name, job.PluginName);
processor.Process(layer);
break;
default:
throw new ArgumentOutOfRangeException(nameof(job.JobType), job.JobType, "Unknown job type");
}
await queueService.MarkJobCompletedAsync(job.Id);
job.Status = JobStatus.Completed;
_logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}",
job.JobType, layer.Name);
}
catch (Exception ex)
{
_logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}",
job.JobType, job.Id, job.LayerName);
// POPRAWKA: tutaj też używaj _scopeFactory
using var scope = _scopeFactory.CreateScope();
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
// Check if it's a retriable error
if (IsRetriableError(ex))
{
await queueService.MarkJobForRetryAsync(job.Id, ex.Message);
job.Status = JobStatus.Retrying;
}
else
{
await queueService.MarkJobFailedAsync(job.Id, ex.Message);
job.Status = JobStatus.Failed;
}
}
}
private static bool IsRetriableError(Exception ex)
{
var message = ex.Message.ToLowerInvariant();
var retriableErrors = new[]
{
"quota", "rate limit", "timeout", "service unavailable",
"internal server error", "bad gateway", "gateway timeout",
"network", "connection"
};
return retriableErrors.Any(error => message.Contains(error)) ||
ex is HttpRequestException ||
ex is TimeoutException;
}
}

View File

@@ -1,155 +0,0 @@
using Microsoft.Extensions.Logging;
using DiunaBI.Core.Models;
using DiunaBI.Core.Interfaces;
using System.Threading.Tasks;
using DiunaBI.Core.Database.Context;
using System;
using System.Linq;
using Microsoft.EntityFrameworkCore;
using System.Collections.Generic;
namespace DiunaBI.Core.Services;
public class JobQueueService : IJobQueueService
{
private readonly AppDbContext _db;
private readonly ILogger<JobQueueService> _logger;
public JobQueueService(AppDbContext db, ILogger<JobQueueService> logger)
{
_db = db;
_logger = logger;
}
public async Task EnqueueJobAsync(QueueJob job)
{
_db.QueueJobs.Add(job);
await _db.SaveChangesAsync();
_logger.LogInformation("JobQueue: Enqueued {JobType} job {JobId} for layer {LayerName} with plugin {PluginName} (priority {Priority})",
job.JobType, job.Id, job.LayerName, job.PluginName, job.Priority);
}
public async Task<QueueJob?> DequeueJobAsync(JobType? jobType = null)
{
var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending);
if (jobType.HasValue)
{
query = query.Where(x => x.JobType == jobType.Value);
}
var job = await query
.OrderBy(x => x.JobType) // Importers first (0), then Processors (1)
.ThenBy(x => x.Priority) // Then by priority
.ThenBy(x => x.CreatedAt) // Then FIFO
.FirstOrDefaultAsync();
if (job != null)
{
job.Status = JobStatus.Running;
job.LastAttemptAt = DateTime.UtcNow;
job.ModifiedAtUtc = DateTime.UtcNow;
await _db.SaveChangesAsync();
_logger.LogDebug("JobQueue: Dequeued {JobType} job {JobId} for layer {LayerName} (priority {Priority})",
job.JobType, job.Id, job.LayerName, job.Priority);
}
return job;
}
public async Task MarkJobCompletedAsync(Guid jobId)
{
var job = await _db.QueueJobs.FindAsync(jobId);
if (job != null)
{
job.Status = JobStatus.Completed;
job.CompletedAt = DateTime.UtcNow;
job.ModifiedAtUtc = DateTime.UtcNow;
await _db.SaveChangesAsync();
_logger.LogInformation("JobQueue: {JobType} job {JobId} completed successfully for layer {LayerName}",
job.JobType, jobId, job.LayerName);
}
}
public async Task MarkJobFailedAsync(Guid jobId, string error)
{
var job = await _db.QueueJobs.FindAsync(jobId);
if (job != null)
{
job.Status = JobStatus.Failed;
job.LastError = error;
job.ModifiedAtUtc = DateTime.UtcNow;
await _db.SaveChangesAsync();
_logger.LogError("JobQueue: {JobType} job {JobId} failed permanently for layer {LayerName}: {Error}",
job.JobType, jobId, job.LayerName, error);
}
}
public async Task MarkJobForRetryAsync(Guid jobId, string error)
{
var job = await _db.QueueJobs.FindAsync(jobId);
if (job != null)
{
job.RetryCount++;
job.LastError = error;
job.ModifiedAtUtc = DateTime.UtcNow;
if (job.RetryCount >= job.MaxRetries)
{
await MarkJobFailedAsync(jobId, $"Max retries ({job.MaxRetries}) exceeded. Last error: {error}");
return;
}
job.Status = JobStatus.Retrying;
await _db.SaveChangesAsync();
// Schedule retry with exponential backoff
var delayMinutes = Math.Pow(2, job.RetryCount);
_ = Task.Delay(TimeSpan.FromMinutes(delayMinutes))
.ContinueWith(async _ =>
{
var retryJob = await _db.QueueJobs.FindAsync(jobId);
if (retryJob?.Status == JobStatus.Retrying)
{
retryJob.Status = JobStatus.Pending;
retryJob.ModifiedAtUtc = DateTime.UtcNow;
await _db.SaveChangesAsync();
_logger.LogWarning("JobQueue: {JobType} job {JobId} re-queued for retry {RetryCount}/{MaxRetries} for layer {LayerName}",
retryJob.JobType, jobId, retryJob.RetryCount, retryJob.MaxRetries, retryJob.LayerName);
}
});
}
}
public async Task<IEnumerable<QueueJob>> GetQueueStatusAsync()
{
return await _db.QueueJobs
.OrderBy(x => x.JobType)
.ThenBy(x => x.Priority)
.ThenBy(x => x.CreatedAt)
.ToListAsync();
}
public async Task<int> GetQueueCountAsync(JobType? jobType = null)
{
var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending);
if (jobType.HasValue)
{
query = query.Where(x => x.JobType == jobType.Value);
}
return await query.CountAsync();
}
public async Task<int> GetRunningJobsCountAsync(JobType jobType)
{
return await _db.QueueJobs.CountAsync(x => x.Status == JobStatus.Running && x.JobType == jobType);
}
}

View File

@@ -21,7 +21,6 @@ public class LayersController : Controller
private readonly IConfiguration _configuration;
private readonly PluginManager _pluginManager;
private readonly ILogger<LayersController> _logger;
private readonly IJobQueueService _queueService;
public LayersController(
AppDbContext db,
@@ -29,8 +28,7 @@ public class LayersController : Controller
GoogleDriveHelper googleDriveHelper,
IConfiguration configuration,
PluginManager pluginManager,
ILogger<LayersController> logger,
IJobQueueService queueService
ILogger<LayersController> logger
)
{
_db = db;
@@ -39,7 +37,6 @@ public class LayersController : Controller
_configuration = configuration;
_pluginManager = pluginManager;
_logger = logger;
_queueService = queueService;
}
[HttpGet]
@@ -241,51 +238,6 @@ public class LayersController : Controller
}
}
[HttpGet]
[Route("AutoImportWithQueue/{apiKey}")]
[AllowAnonymous]
public IActionResult AutoImportWithQueue(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("AutoImportQueue: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
)
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToList();
if (importWorkerLayers.Count == 0)
{
_logger.LogInformation("AutoImportQueue: No layers to import");
return Ok();
}
_logger.LogInformation("AutoImportQueue: Found {LayerCount} layers to queue", importWorkerLayers.Count);
foreach (var importWorker in importWorkerLayers)
{
try
{
// Queue job implementation would go here
_logger.LogDebug("AutoImportQueue: Queued layer {LayerName} ({LayerId})", importWorker.Name, importWorker.Id);
}
catch (Exception e)
{
_logger.LogError(e, "AutoImportQueue: Error while adding job for layer {LayerName} ({LayerId})",
importWorker.Name, importWorker.Id);
}
}
return Ok();
}
[HttpGet]
[Route("ProcessQueue/{apiKey}")]
[AllowAnonymous]
@@ -679,42 +631,6 @@ public class LayersController : Controller
[HttpGet]
[Route("CheckProcessors")]
[AllowAnonymous]
public IActionResult CheckProcessors()
{
// get
/*
var importWorkerLayers = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "ImportType" && y.Desc1 == "Import-D3")
)
.OrderByDescending(x => x.CreatedAt)
.AsNoTracking()
.ToList();
foreach (var importWorker in importWorkerLayers)
{
var record = new Record
{
Id = Guid.NewGuid(),
LayerId = importWorker.Id,
Code = "Plugin",
Desc1 = "Morska.Import.D3",
CreatedAt = DateTime.UtcNow,
ModifiedAt = DateTime.UtcNow,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
//_db.Records.Add(record);
}
//_db.SaveChanges();
*/
return Ok();
}
private static void WriteToConsole(params string[] messages)
{
@@ -809,278 +725,4 @@ public class LayersController : Controller
throw;
}
}
[HttpGet]
[Route("EnqueueImportWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueImportWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueImportWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
).Take(5);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var importWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueImportWorkers: Found {LayerCount} import worker layers to queue{LayerFilter}",
importWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (importWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No import workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var importWorker in importWorkerLayers)
{
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
importWorker.Name, importWorker.Id);
continue;
}
// Check if plugin exists
var importer = _pluginManager.GetImporter(pluginName);
if (importer == null)
{
_logger.LogWarning("EnqueueImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, importWorker.Name, importWorker.Id);
continue;
}
var job = new QueueJob
{
LayerId = importWorker.Id,
LayerName = importWorker.Name ?? "Unknown",
PluginName = pluginName,
JobType = JobType.Import,
Priority = 0, // All imports have same priority
MaxRetries = 5,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
importWorker.Name, importWorker.Id, pluginName);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} import jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = importWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueImportWorkers: Error queuing import workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("EnqueueProcessWorkers/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> EnqueueProcessWorkers(string apiKey, [FromQuery] Guid? layerId = null)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("EnqueueProcessWorkers: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var query = _db.Layers
.Include(x => x.Records)
.Where(x =>
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ProcessWorker") &&
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True") &&
!x.IsDeleted && !x.IsCancelled
).Take(5);
// If specific layerId is provided, filter to that layer only
if (layerId.HasValue)
{
query = query.Where(x => x.Id == layerId.Value);
}
var processWorkerLayers = await query
.OrderBy(x => x.CreatedAt)
.AsNoTracking()
.ToListAsync();
_logger.LogInformation("EnqueueProcessWorkers: Found {LayerCount} process worker layers to queue{LayerFilter}",
processWorkerLayers.Count, layerId.HasValue ? $" (filtered by LayerId: {layerId})" : "");
if (processWorkerLayers.Count == 0)
{
return Ok(new
{
Message = "No process workers found to queue",
QueuedJobs = 0,
TotalQueueSize = await _queueService.GetQueueCountAsync(),
SkippedLayers = 0
});
}
int queuedCount = 0;
foreach (var processWorker in processWorkerLayers)
{
// POPRAWIONE: Używaj Plugin zamiast ProcessorType
var pluginName = processWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
if (string.IsNullOrEmpty(pluginName))
{
_logger.LogWarning("EnqueueProcessWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
processWorker.Name, processWorker.Id);
continue;
}
// POPRAWIONE: Sprawdź czy processor istnieje przez pluginName
var processor = _pluginManager.GetProcessor(pluginName);
if (processor == null)
{
_logger.LogWarning("EnqueueProcessWorkers: Processor {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
pluginName, processWorker.Name, processWorker.Id);
continue;
}
// Get priority from ProcessWorker record, default to 10 if not found
var priorityStr = processWorker.Records!.FirstOrDefault(x => x.Code == "Priority")?.Desc1;
var priority = 10; // Default priority
if (!string.IsNullOrEmpty(priorityStr) && int.TryParse(priorityStr, out var parsedPriority))
{
priority = parsedPriority;
}
var job = new QueueJob
{
LayerId = processWorker.Id,
LayerName = processWorker.Name ?? "Unknown",
PluginName = pluginName, // POPRAWIONE: Używaj pluginName bezpośrednio
JobType = JobType.Process,
Priority = priority,
MaxRetries = 3,
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
};
await _queueService.EnqueueJobAsync(job);
queuedCount++;
_logger.LogDebug("EnqueueProcessWorkers: Queued process job for layer {LayerName} ({LayerId}) with plugin {PluginName}, priority {Priority}",
processWorker.Name, processWorker.Id, pluginName, priority);
}
var totalQueueSize = await _queueService.GetQueueCountAsync();
_logger.LogInformation("EnqueueProcessWorkers: Successfully queued {QueuedCount} process jobs. Total queue size: {QueueSize}",
queuedCount, totalQueueSize);
return Ok(new
{
Message = $"Queued {queuedCount} process jobs",
QueuedJobs = queuedCount,
TotalQueueSize = totalQueueSize,
SkippedLayers = processWorkerLayers.Count - queuedCount
});
}
catch (Exception e)
{
_logger.LogError(e, "EnqueueProcessWorkers: Error queuing process workers");
return BadRequest(e.ToString());
}
}
[HttpGet]
[Route("RunQueueJobs/{apiKey}")]
[AllowAnonymous]
public async Task<IActionResult> RunQueueJobs(string apiKey)
{
if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
{
_logger.LogWarning("RunQueueJobs: Unauthorized request with apiKey {ApiKey}", apiKey);
return Unauthorized();
}
try
{
var queueSize = await _queueService.GetQueueCountAsync();
if (queueSize == 0)
{
return Ok(new
{
Message = "Queue is empty",
QueueSize = 0,
Status = "No jobs to process"
});
}
_logger.LogInformation("RunQueueJobs: Triggering queue processing for {QueueSize} jobs", queueSize);
// ZMIEŃ NA DOSTĘP PRZEZ IHostedService:
var hostedServices = HttpContext.RequestServices.GetServices<IHostedService>();
var queueProcessor = hostedServices.OfType<JobQueueProcessor>().FirstOrDefault();
if (queueProcessor == null)
{
_logger.LogError("RunQueueJobs: JobQueueProcessor not found");
return BadRequest("JobQueueProcessor not found");
}
queueProcessor.TriggerProcessing();
return Ok(new
{
Message = $"Queue processing triggered for {queueSize} jobs",
QueueSize = queueSize,
Status = "Processing started in background"
});
}
catch (Exception e)
{
_logger.LogError(e, "RunQueueJobs: Error triggering queue processing");
return BadRequest(e.ToString());
}
}
}

View File

@@ -69,10 +69,6 @@ builder.Services.AddAuthentication(options =>
};
});
// Queue services
builder.Services.AddScoped<IJobQueueService, JobQueueService>();
builder.Services.AddHostedService<JobQueueProcessor>(); // ✅ GOOD - with proper scope factory
// Google Sheets dependencies
Console.WriteLine("Adding Google Sheets dependencies...");
builder.Services.AddSingleton<GoogleSheetsHelper>();
@@ -142,9 +138,7 @@ app.Use(async (context, next) =>
if (token.Length > 0
&& !context.Request.Path.ToString().Contains("getForPowerBI")
&& !context.Request.Path.ToString().Contains("getConfiguration")
&& !context.Request.Path.ToString().Contains("DataInbox/Add")
&& !context.Request.Path.ToString().Contains("AddPluginName") // TODO: Remove this
&& !context.Request.Path.ToString().Contains("GetImportWorkers"))
&& !context.Request.Path.ToString().Contains("DataInbox/Add"))
{
var handler = new JwtSecurityTokenHandler();
var data = handler.ReadJwtToken(token.Split(' ')[1]);