Move Databese stuff into Core and remove DiunaBI.Database Project
This commit is contained in:
@@ -2,7 +2,7 @@
|
|||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Database.Context;
|
namespace DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
|
public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
|
||||||
{
|
{
|
||||||
@@ -4,7 +4,7 @@ using Microsoft.EntityFrameworkCore;
|
|||||||
using Microsoft.EntityFrameworkCore.Design;
|
using Microsoft.EntityFrameworkCore.Design;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
|
|
||||||
namespace DiunaBI.Database.Context;
|
namespace DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbContext>
|
public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbContext>
|
||||||
{
|
{
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ using Microsoft.EntityFrameworkCore.Metadata;
|
|||||||
using Microsoft.EntityFrameworkCore.Migrations;
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -5,7 +5,7 @@ using Microsoft.EntityFrameworkCore.Infrastructure;
|
|||||||
using Microsoft.EntityFrameworkCore.Metadata;
|
using Microsoft.EntityFrameworkCore.Metadata;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -9,7 +9,21 @@
|
|||||||
<PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3490" />
|
<PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3490" />
|
||||||
<PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3525" />
|
<PackageReference Include="Google.Apis.Sheets.v4" Version="1.68.0.3525" />
|
||||||
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
|
<PackageReference Include="Google.Apis.Auth" Version="1.68.0" />
|
||||||
<PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
|
<PackageReference Include="AngouriMath" Version="1.4.0-preview.3" />
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.0">
|
||||||
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
</PackageReference>
|
||||||
|
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.0">
|
||||||
|
<PrivateAssets>all</PrivateAssets>
|
||||||
|
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||||
|
</PackageReference>
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
18
src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs
Normal file
18
src/Backend/DiunaBI.Core/Interfaces/IJobQueueService.cs
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Interfaces;
|
||||||
|
|
||||||
|
public interface IJobQueueService
|
||||||
|
{
|
||||||
|
Task EnqueueJobAsync(QueueJob job);
|
||||||
|
Task<QueueJob?> DequeueJobAsync(JobType? jobType = null);
|
||||||
|
Task MarkJobCompletedAsync(Guid jobId);
|
||||||
|
Task MarkJobFailedAsync(Guid jobId, string error);
|
||||||
|
Task MarkJobForRetryAsync(Guid jobId, string error);
|
||||||
|
Task<IEnumerable<QueueJob>> GetQueueStatusAsync();
|
||||||
|
Task<int> GetQueueCountAsync(JobType? jobType = null);
|
||||||
|
Task<int> GetRunningJobsCountAsync(JobType jobType);
|
||||||
|
}
|
||||||
@@ -3,27 +3,68 @@ using System.ComponentModel.DataAnnotations;
|
|||||||
|
|
||||||
namespace DiunaBI.Core.Models;
|
namespace DiunaBI.Core.Models;
|
||||||
|
|
||||||
public enum JobStatus
|
public class QueueJob
|
||||||
{
|
{
|
||||||
New,
|
[Key]
|
||||||
Failed,
|
public Guid Id { get; set; } = Guid.NewGuid();
|
||||||
Success
|
|
||||||
|
[Required]
|
||||||
|
public Guid LayerId { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
[MaxLength(200)]
|
||||||
|
public string LayerName { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
[MaxLength(100)]
|
||||||
|
public string PluginName { get; set; } = string.Empty;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public JobType JobType { get; set; }
|
||||||
|
|
||||||
|
public int Priority { get; set; } = 0; // 0 = highest priority
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
||||||
|
|
||||||
|
public int RetryCount { get; set; } = 0;
|
||||||
|
|
||||||
|
public int MaxRetries { get; set; } = 5;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public JobStatus Status { get; set; } = JobStatus.Pending;
|
||||||
|
|
||||||
|
[MaxLength(1000)]
|
||||||
|
public string? LastError { get; set; }
|
||||||
|
|
||||||
|
public DateTime? LastAttemptAt { get; set; }
|
||||||
|
|
||||||
|
public DateTime? CompletedAt { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public Guid CreatedById { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow;
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public Guid ModifiedById { get; set; }
|
||||||
|
|
||||||
|
[Required]
|
||||||
|
public DateTime ModifiedAtUtc { get; set; } = DateTime.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum JobType
|
public enum JobType
|
||||||
{
|
{
|
||||||
ImportWorker,
|
Import = 0,
|
||||||
ProcessWorker
|
Process = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
public class QueueJob
|
public enum JobStatus
|
||||||
{
|
{
|
||||||
[Key] public Guid Id { get; set; }
|
Pending,
|
||||||
[Required] public Guid LayerId { get; set; }
|
Running,
|
||||||
[Required] public int Attempts { get; set; }
|
Completed,
|
||||||
[Required] public JobStatus Status { get; set; } = JobStatus.New;
|
Failed,
|
||||||
[Required] public JobType Type { get; set; } = JobType.ImportWorker;
|
Retrying
|
||||||
public string Message { get; set; } = string.Empty;
|
|
||||||
[Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
|
||||||
[Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow;
|
|
||||||
}
|
}
|
||||||
155
src/Backend/DiunaBI.Core/Services/JobQueueService.cs
Normal file
155
src/Backend/DiunaBI.Core/Services/JobQueueService.cs
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Interfaces;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using System;
|
||||||
|
using System.Linq;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Services;
|
||||||
|
|
||||||
|
public class JobQueueService : IJobQueueService
|
||||||
|
{
|
||||||
|
private readonly AppDbContext _db;
|
||||||
|
private readonly ILogger<JobQueueService> _logger;
|
||||||
|
|
||||||
|
public JobQueueService(AppDbContext db, ILogger<JobQueueService> logger)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task EnqueueJobAsync(QueueJob job)
|
||||||
|
{
|
||||||
|
_db.QueueJobs.Add(job);
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueue: Enqueued {JobType} job {JobId} for layer {LayerName} with plugin {PluginName} (priority {Priority})",
|
||||||
|
job.JobType, job.Id, job.LayerName, job.PluginName, job.Priority);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<QueueJob?> DequeueJobAsync(JobType? jobType = null)
|
||||||
|
{
|
||||||
|
var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending);
|
||||||
|
|
||||||
|
if (jobType.HasValue)
|
||||||
|
{
|
||||||
|
query = query.Where(x => x.JobType == jobType.Value);
|
||||||
|
}
|
||||||
|
|
||||||
|
var job = await query
|
||||||
|
.OrderBy(x => x.JobType) // Importers first (0), then Processors (1)
|
||||||
|
.ThenBy(x => x.Priority) // Then by priority
|
||||||
|
.ThenBy(x => x.CreatedAt) // Then FIFO
|
||||||
|
.FirstOrDefaultAsync();
|
||||||
|
|
||||||
|
if (job != null)
|
||||||
|
{
|
||||||
|
job.Status = JobStatus.Running;
|
||||||
|
job.LastAttemptAt = DateTime.UtcNow;
|
||||||
|
job.ModifiedAtUtc = DateTime.UtcNow;
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_logger.LogDebug("JobQueue: Dequeued {JobType} job {JobId} for layer {LayerName} (priority {Priority})",
|
||||||
|
job.JobType, job.Id, job.LayerName, job.Priority);
|
||||||
|
}
|
||||||
|
|
||||||
|
return job;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task MarkJobCompletedAsync(Guid jobId)
|
||||||
|
{
|
||||||
|
var job = await _db.QueueJobs.FindAsync(jobId);
|
||||||
|
if (job != null)
|
||||||
|
{
|
||||||
|
job.Status = JobStatus.Completed;
|
||||||
|
job.CompletedAt = DateTime.UtcNow;
|
||||||
|
job.ModifiedAtUtc = DateTime.UtcNow;
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueue: {JobType} job {JobId} completed successfully for layer {LayerName}",
|
||||||
|
job.JobType, jobId, job.LayerName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task MarkJobFailedAsync(Guid jobId, string error)
|
||||||
|
{
|
||||||
|
var job = await _db.QueueJobs.FindAsync(jobId);
|
||||||
|
if (job != null)
|
||||||
|
{
|
||||||
|
job.Status = JobStatus.Failed;
|
||||||
|
job.LastError = error;
|
||||||
|
job.ModifiedAtUtc = DateTime.UtcNow;
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_logger.LogError("JobQueue: {JobType} job {JobId} failed permanently for layer {LayerName}: {Error}",
|
||||||
|
job.JobType, jobId, job.LayerName, error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task MarkJobForRetryAsync(Guid jobId, string error)
|
||||||
|
{
|
||||||
|
var job = await _db.QueueJobs.FindAsync(jobId);
|
||||||
|
if (job != null)
|
||||||
|
{
|
||||||
|
job.RetryCount++;
|
||||||
|
job.LastError = error;
|
||||||
|
job.ModifiedAtUtc = DateTime.UtcNow;
|
||||||
|
|
||||||
|
if (job.RetryCount >= job.MaxRetries)
|
||||||
|
{
|
||||||
|
await MarkJobFailedAsync(jobId, $"Max retries ({job.MaxRetries}) exceeded. Last error: {error}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
job.Status = JobStatus.Retrying;
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
// Schedule retry with exponential backoff
|
||||||
|
var delayMinutes = Math.Pow(2, job.RetryCount);
|
||||||
|
_ = Task.Delay(TimeSpan.FromMinutes(delayMinutes))
|
||||||
|
.ContinueWith(async _ =>
|
||||||
|
{
|
||||||
|
var retryJob = await _db.QueueJobs.FindAsync(jobId);
|
||||||
|
if (retryJob?.Status == JobStatus.Retrying)
|
||||||
|
{
|
||||||
|
retryJob.Status = JobStatus.Pending;
|
||||||
|
retryJob.ModifiedAtUtc = DateTime.UtcNow;
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_logger.LogWarning("JobQueue: {JobType} job {JobId} re-queued for retry {RetryCount}/{MaxRetries} for layer {LayerName}",
|
||||||
|
retryJob.JobType, jobId, retryJob.RetryCount, retryJob.MaxRetries, retryJob.LayerName);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<IEnumerable<QueueJob>> GetQueueStatusAsync()
|
||||||
|
{
|
||||||
|
return await _db.QueueJobs
|
||||||
|
.OrderBy(x => x.JobType)
|
||||||
|
.ThenBy(x => x.Priority)
|
||||||
|
.ThenBy(x => x.CreatedAt)
|
||||||
|
.ToListAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<int> GetQueueCountAsync(JobType? jobType = null)
|
||||||
|
{
|
||||||
|
var query = _db.QueueJobs.Where(x => x.Status == JobStatus.Pending);
|
||||||
|
|
||||||
|
if (jobType.HasValue)
|
||||||
|
{
|
||||||
|
query = query.Where(x => x.JobType == jobType.Value);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await query.CountAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<int> GetRunningJobsCountAsync(JobType jobType)
|
||||||
|
{
|
||||||
|
return await _db.QueueJobs.CountAsync(x => x.Status == JobStatus.Running && x.JobType == jobType);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<!-- EF Core packages -->
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="8.0.0">
|
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
|
||||||
<PrivateAssets>all</PrivateAssets>
|
|
||||||
</PackageReference>
|
|
||||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.0">
|
|
||||||
<PrivateAssets>all</PrivateAssets>
|
|
||||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
|
||||||
</PackageReference>
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
</Project>
|
|
||||||
@@ -12,6 +12,5 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
||||||
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
@@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4;
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ using System.Text.Json;
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4;
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4;
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Importers;
|
namespace DiunaBI.Plugins.Morska.Importers;
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using Google.Apis.Sheets.v4;
|
|||||||
using Google.Apis.Sheets.v4.Data;
|
using Google.Apis.Sheets.v4.Data;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using DiunaBI.Core.Services.Calculations;
|
using DiunaBI.Core.Services.Calculations;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ using Google.Apis.Sheets.v4;
|
|||||||
using Google.Apis.Sheets.v4.Data;
|
using Google.Apis.Sheets.v4.Data;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services.Calculations;
|
using DiunaBI.Core.Services.Calculations;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using DiunaBI.Core.Services.Calculations;
|
using DiunaBI.Core.Services.Calculations;
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ using Google.Apis.Sheets.v4.Data;
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
|
||||||
namespace DiunaBI.Plugins.Morska.Processors;
|
namespace DiunaBI.Plugins.Morska.Processors;
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ using System.IdentityModel.Tokens.Jwt;
|
|||||||
using System.Security.Claims;
|
using System.Security.Claims;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|
||||||
[ApiController]
|
[ApiController]
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ using Microsoft.AspNetCore.Authorization;
|
|||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ using Microsoft.AspNetCore.Authorization;
|
|||||||
using Microsoft.AspNetCore.Mvc;
|
using Microsoft.AspNetCore.Mvc;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Cloud.Firestore;
|
using Google.Cloud.Firestore;
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,6 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
<ProjectReference Include="..\DiunaBI.Core\DiunaBI.Core.csproj" />
|
||||||
<ProjectReference Include="..\DiunaBI.Database\DiunaBI.Database.csproj" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ using Microsoft.IdentityModel.Tokens;
|
|||||||
using System.IdentityModel.Tokens.Jwt;
|
using System.IdentityModel.Tokens.Jwt;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using DiunaBI.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 17
|
# Visual Studio Version 17
|
||||||
VisualStudioVersion = 17.0.31903.59
|
VisualStudioVersion = 17.0.31903.59
|
||||||
@@ -9,8 +9,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Core", "DiunaBI.Cor
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Plugins.Morska", "DiunaBI.Plugins.Morska\DiunaBI.Plugins.Morska.csproj", "{B5416A3F-550A-468D-852F-20B24243FD68}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiunaBI.Database", "DiunaBI.Database\DiunaBI.Database.csproj", "{8C346BEA-A209-4E8F-A6BF-70B42D9106C8}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
|
|||||||
Reference in New Issue
Block a user