JobQueue for import layers
This commit is contained in:
Michał Zieliński
@@ -21,14 +21,4 @@ public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(op
|
||||
x.SourceId
|
||||
});
|
||||
}
|
||||
|
||||
private static readonly LoggerFactory MyLoggerFactory =
|
||||
new(new[] {
|
||||
new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider()
|
||||
});
|
||||
|
||||
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
|
||||
{
|
||||
optionsBuilder.UseLoggerFactory(MyLoggerFactory);
|
||||
}
|
||||
}
|
||||
@@ -11,17 +11,17 @@ public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbConte
|
||||
public AppDbContext CreateDbContext(string[] args)
|
||||
{
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI")) // ✅ Poprawna ścieżka
|
||||
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI"))
|
||||
.AddJsonFile("appsettings.json", optional: false)
|
||||
.AddJsonFile("appsettings.Development.json", optional: true)
|
||||
.Build();
|
||||
|
||||
var optionsBuilder = new DbContextOptionsBuilder<AppDbContext>();
|
||||
|
||||
var connectionString = configuration.GetConnectionString("DefaultConnection");
|
||||
var connectionString = configuration.GetConnectionString("SQLDatabase");
|
||||
if (string.IsNullOrEmpty(connectionString))
|
||||
{
|
||||
throw new InvalidOperationException("Connection string 'DefaultConnection' not found in appsettings.json");
|
||||
throw new InvalidOperationException("Connection string 'SQLDatabase' not found in appsettings.json");
|
||||
}
|
||||
|
||||
optionsBuilder.UseSqlServer(connectionString);
|
||||
|
||||
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
@@ -0,0 +1,415 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using DiunaBI.Core.Database.Context;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace DiunaBI.Core.Migrations
|
||||
{
|
||||
[DbContext(typeof(AppDbContext))]
|
||||
[Migration("20250607084540_QueueJobRefactor")]
|
||||
partial class QueueJobRefactor
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "8.0.0")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||
|
||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<string>("Data")
|
||||
.IsRequired()
|
||||
.HasMaxLength(2147483647)
|
||||
.HasColumnType("nvarchar(max)");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.Property<string>("Source")
|
||||
.IsRequired()
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("DataInbox");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("CreatedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<bool>("IsCancelled")
|
||||
.HasColumnType("bit");
|
||||
|
||||
b.Property<bool>("IsDeleted")
|
||||
.HasColumnType("bit");
|
||||
|
||||
b.Property<DateTime>("ModifiedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("ModifiedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("Name")
|
||||
.IsRequired()
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.Property<int>("Number")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<Guid?>("ParentId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<int>("Type")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("CreatedById");
|
||||
|
||||
b.HasIndex("ModifiedById");
|
||||
|
||||
b.HasIndex("ParentId");
|
||||
|
||||
b.ToTable("Layers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||
{
|
||||
b.Property<Guid>("LayerId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<Guid>("SourceId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.HasKey("LayerId", "SourceId");
|
||||
|
||||
b.HasIndex("SourceId");
|
||||
|
||||
b.ToTable("ProcessSources");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime?>("CompletedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<DateTime>("CreatedAtUtc")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("CreatedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<int>("JobType")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTime?>("LastAttemptAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<string>("LastError")
|
||||
.HasMaxLength(1000)
|
||||
.HasColumnType("nvarchar(1000)");
|
||||
|
||||
b.Property<Guid>("LayerId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("LayerName")
|
||||
.IsRequired()
|
||||
.HasMaxLength(200)
|
||||
.HasColumnType("nvarchar(200)");
|
||||
|
||||
b.Property<int>("MaxRetries")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTime>("ModifiedAtUtc")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("ModifiedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("PluginName")
|
||||
.IsRequired()
|
||||
.HasMaxLength(100)
|
||||
.HasColumnType("nvarchar(100)");
|
||||
|
||||
b.Property<int>("Priority")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<int>("RetryCount")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<int>("Status")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("QueueJobs");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("Code")
|
||||
.IsRequired()
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("CreatedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("Desc1")
|
||||
.HasMaxLength(10000)
|
||||
.HasColumnType("nvarchar(max)");
|
||||
|
||||
b.Property<bool>("IsDeleted")
|
||||
.HasColumnType("bit");
|
||||
|
||||
b.Property<Guid>("LayerId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime>("ModifiedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("ModifiedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<double?>("Value1")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value10")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value11")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value12")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value13")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value14")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value15")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value16")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value17")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value18")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value19")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value2")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value20")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value21")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value22")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value23")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value24")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value25")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value26")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value27")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value28")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value29")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value3")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value30")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value31")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value32")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value4")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value5")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value6")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value7")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value8")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.Property<double?>("Value9")
|
||||
.HasColumnType("float");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.HasIndex("CreatedById");
|
||||
|
||||
b.HasIndex("LayerId");
|
||||
|
||||
b.HasIndex("ModifiedById");
|
||||
|
||||
b.ToTable("Records");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<string>("Email")
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.Property<string>("UserName")
|
||||
.HasMaxLength(50)
|
||||
.HasColumnType("nvarchar(50)");
|
||||
|
||||
b.HasKey("Id");
|
||||
|
||||
b.ToTable("Users");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("CreatedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("ModifiedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||
.WithMany()
|
||||
.HasForeignKey("ParentId");
|
||||
|
||||
b.Navigation("CreatedBy");
|
||||
|
||||
b.Navigation("ModifiedBy");
|
||||
|
||||
b.Navigation("Parent");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||
{
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||
.WithMany()
|
||||
.HasForeignKey("SourceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.Navigation("Source");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||
{
|
||||
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("CreatedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||
.WithMany("Records")
|
||||
.HasForeignKey("LayerId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("ModifiedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.Navigation("CreatedBy");
|
||||
|
||||
b.Navigation("ModifiedBy");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.Navigation("Records");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace DiunaBI.Core.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class QueueJobRefactor : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "Message",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "Type",
|
||||
table: "QueueJobs",
|
||||
newName: "RetryCount");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "ModifiedAt",
|
||||
table: "QueueJobs",
|
||||
newName: "ModifiedAtUtc");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "Attempts",
|
||||
table: "QueueJobs",
|
||||
newName: "Priority");
|
||||
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
name: "CompletedAt",
|
||||
table: "QueueJobs",
|
||||
type: "datetime2",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
name: "CreatedAtUtc",
|
||||
table: "QueueJobs",
|
||||
type: "datetime2",
|
||||
nullable: false,
|
||||
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
|
||||
|
||||
migrationBuilder.AddColumn<Guid>(
|
||||
name: "CreatedById",
|
||||
table: "QueueJobs",
|
||||
type: "uniqueidentifier",
|
||||
nullable: false,
|
||||
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||
|
||||
migrationBuilder.AddColumn<int>(
|
||||
name: "JobType",
|
||||
table: "QueueJobs",
|
||||
type: "int",
|
||||
nullable: false,
|
||||
defaultValue: 0);
|
||||
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
name: "LastAttemptAt",
|
||||
table: "QueueJobs",
|
||||
type: "datetime2",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "LastError",
|
||||
table: "QueueJobs",
|
||||
type: "nvarchar(1000)",
|
||||
maxLength: 1000,
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "LayerName",
|
||||
table: "QueueJobs",
|
||||
type: "nvarchar(200)",
|
||||
maxLength: 200,
|
||||
nullable: false,
|
||||
defaultValue: "");
|
||||
|
||||
migrationBuilder.AddColumn<int>(
|
||||
name: "MaxRetries",
|
||||
table: "QueueJobs",
|
||||
type: "int",
|
||||
nullable: false,
|
||||
defaultValue: 0);
|
||||
|
||||
migrationBuilder.AddColumn<Guid>(
|
||||
name: "ModifiedById",
|
||||
table: "QueueJobs",
|
||||
type: "uniqueidentifier",
|
||||
nullable: false,
|
||||
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "PluginName",
|
||||
table: "QueueJobs",
|
||||
type: "nvarchar(100)",
|
||||
maxLength: 100,
|
||||
nullable: false,
|
||||
defaultValue: "");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "CompletedAt",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "CreatedAtUtc",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "CreatedById",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "JobType",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "LastAttemptAt",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "LastError",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "LayerName",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "MaxRetries",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "ModifiedById",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "PluginName",
|
||||
table: "QueueJobs");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "RetryCount",
|
||||
table: "QueueJobs",
|
||||
newName: "Type");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "Priority",
|
||||
table: "QueueJobs",
|
||||
newName: "Attempts");
|
||||
|
||||
migrationBuilder.RenameColumn(
|
||||
name: "ModifiedAtUtc",
|
||||
table: "QueueJobs",
|
||||
newName: "ModifiedAt");
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "Message",
|
||||
table: "QueueJobs",
|
||||
type: "nvarchar(max)",
|
||||
nullable: false,
|
||||
defaultValue: "");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,10 @@
|
||||
// <auto-generated />
|
||||
using System;
|
||||
using DiunaBI.Core.Database.Context;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
using DiunaBI.Core.Models;
|
||||
using DiunaBI.Core.Database.Context;
|
||||
|
||||
#nullable disable
|
||||
|
||||
@@ -18,12 +17,12 @@ namespace DiunaBI.Core.Migrations
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder
|
||||
.HasAnnotation("ProductVersion", "9.0.0")
|
||||
.HasAnnotation("ProductVersion", "8.0.0")
|
||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||
|
||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.DataInbox", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
@@ -52,7 +51,7 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("DataInbox");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
@@ -101,7 +100,7 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("Layers");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||
{
|
||||
b.Property<Guid>("LayerId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
@@ -116,32 +115,63 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("ProcessSources");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<int>("Attempts")
|
||||
.HasColumnType("int");
|
||||
b.Property<DateTime?>("CompletedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<DateTime>("CreatedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<DateTime>("CreatedAtUtc")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("CreatedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<int>("JobType")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<DateTime?>("LastAttemptAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<string>("LastError")
|
||||
.HasMaxLength(1000)
|
||||
.HasColumnType("nvarchar(1000)");
|
||||
|
||||
b.Property<Guid>("LayerId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("Message")
|
||||
b.Property<string>("LayerName")
|
||||
.IsRequired()
|
||||
.HasColumnType("nvarchar(max)");
|
||||
.HasMaxLength(200)
|
||||
.HasColumnType("nvarchar(200)");
|
||||
|
||||
b.Property<DateTime>("ModifiedAt")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<int>("Status")
|
||||
b.Property<int>("MaxRetries")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<int>("Type")
|
||||
b.Property<DateTime>("ModifiedAtUtc")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("ModifiedById")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("PluginName")
|
||||
.IsRequired()
|
||||
.HasMaxLength(100)
|
||||
.HasColumnType("nvarchar(100)");
|
||||
|
||||
b.Property<int>("Priority")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<int>("RetryCount")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.Property<int>("Status")
|
||||
.HasColumnType("int");
|
||||
|
||||
b.HasKey("Id");
|
||||
@@ -149,7 +179,7 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("QueueJobs");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
@@ -289,7 +319,7 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("Records");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.User", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||
{
|
||||
b.Property<Guid>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
@@ -311,21 +341,21 @@ namespace DiunaBI.Core.Migrations
|
||||
b.ToTable("Users");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
||||
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("CreatedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
||||
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("ModifiedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("WebAPI.Models.Layer", "Parent")
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||
.WithMany()
|
||||
.HasForeignKey("ParentId");
|
||||
|
||||
@@ -336,9 +366,9 @@ namespace DiunaBI.Core.Migrations
|
||||
b.Navigation("Parent");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||
{
|
||||
b.HasOne("WebAPI.Models.Layer", "Source")
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||
.WithMany()
|
||||
.HasForeignKey("SourceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
@@ -347,21 +377,21 @@ namespace DiunaBI.Core.Migrations
|
||||
b.Navigation("Source");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||
{
|
||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
||||
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("CreatedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("WebAPI.Models.Layer", null)
|
||||
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||
.WithMany("Records")
|
||||
.HasForeignKey("LayerId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
||||
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||
.WithMany()
|
||||
.HasForeignKey("ModifiedById")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
@@ -372,7 +402,7 @@ namespace DiunaBI.Core.Migrations
|
||||
b.Navigation("ModifiedBy");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
||||
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||
{
|
||||
b.Navigation("Records");
|
||||
});
|
||||
|
||||
202
src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs
Normal file
202
src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs
Normal file
@@ -0,0 +1,202 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using DiunaBI.Core.Models;
|
||||
using DiunaBI.Core.Interfaces;
|
||||
using DiunaBI.Core.Database.Context;
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using System.Net.Http;
|
||||
using System.Linq;
|
||||
|
||||
namespace DiunaBI.Core.Services;
|
||||
|
||||
public class JobQueueProcessor : BackgroundService
|
||||
{
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly ILogger<JobQueueProcessor> _logger;
|
||||
|
||||
public JobQueueProcessor(
|
||||
IServiceProvider serviceProvider,
|
||||
ILogger<JobQueueProcessor> logger)
|
||||
{
|
||||
_serviceProvider = serviceProvider;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
_logger.LogInformation("JobQueueProcessor: Started");
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||
|
||||
// First process all imports (they run sequentially due to Google Sheets API limits)
|
||||
await ProcessJobType(queueService, JobType.Import, maxConcurrency: 1, stoppingToken);
|
||||
|
||||
// Then process processors (can run in parallel within same priority)
|
||||
await ProcessJobType(queueService, JobType.Process, maxConcurrency: 3, stoppingToken);
|
||||
|
||||
// Wait before next cycle
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_logger.LogInformation("JobQueueProcessor: Cancellation requested");
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "JobQueueProcessor: Unexpected error in queue processor");
|
||||
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("JobQueueProcessor: Stopped");
|
||||
}
|
||||
|
||||
private async Task ProcessJobType(IJobQueueService queueService, JobType jobType, int maxConcurrency, CancellationToken cancellationToken)
|
||||
{
|
||||
var runningJobs = await queueService.GetRunningJobsCountAsync(jobType);
|
||||
|
||||
// Don't start new jobs if we're at max concurrency
|
||||
if (runningJobs >= maxConcurrency)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var job = await queueService.DequeueJobAsync(jobType);
|
||||
if (job == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})",
|
||||
job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority);
|
||||
|
||||
// Process job asynchronously to allow parallel processing of processors
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessJobAsync(job, cancellationToken);
|
||||
|
||||
// Add delay between imports to respect Google Sheets API limits
|
||||
if (job.JobType == JobType.Import)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "JobQueueProcessor: Error in background job processing for {JobType} job {JobId}",
|
||||
job.JobType, job.Id);
|
||||
}
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var dbContext = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var pluginManager = scope.ServiceProvider.GetRequiredService<PluginManager>();
|
||||
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||
|
||||
// Get the layer with records
|
||||
var layer = await dbContext.Layers
|
||||
.Include(x => x.Records)
|
||||
.FirstOrDefaultAsync(x => x.Id == job.LayerId && !x.IsDeleted, cancellationToken);
|
||||
|
||||
if (layer == null)
|
||||
{
|
||||
_logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId);
|
||||
await queueService.MarkJobFailedAsync(job.Id, "Layer not found");
|
||||
return;
|
||||
}
|
||||
|
||||
// Process based on job type
|
||||
switch (job.JobType)
|
||||
{
|
||||
case JobType.Import:
|
||||
var importer = pluginManager.GetImporter(job.PluginName);
|
||||
if (importer == null)
|
||||
{
|
||||
_logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName);
|
||||
await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("JobQueueProcessor: Executing import for layer {LayerName} with plugin {PluginName}",
|
||||
layer.Name, job.PluginName);
|
||||
|
||||
importer.Import(layer);
|
||||
break;
|
||||
|
||||
case JobType.Process:
|
||||
var processor = pluginManager.GetProcessor(job.PluginName);
|
||||
if (processor == null)
|
||||
{
|
||||
_logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName);
|
||||
await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with plugin {PluginName}",
|
||||
layer.Name, job.PluginName);
|
||||
|
||||
processor.Process(layer);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(job.JobType), job.JobType, "Unknown job type");
|
||||
}
|
||||
|
||||
await queueService.MarkJobCompletedAsync(job.Id);
|
||||
|
||||
_logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}",
|
||||
job.JobType, layer.Name);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}",
|
||||
job.JobType, job.Id, job.LayerName);
|
||||
|
||||
using var scope = _serviceProvider.CreateScope();
|
||||
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||
|
||||
// Check if it's a retriable error
|
||||
if (IsRetriableError(ex))
|
||||
{
|
||||
await queueService.MarkJobForRetryAsync(job.Id, ex.Message);
|
||||
}
|
||||
else
|
||||
{
|
||||
await queueService.MarkJobFailedAsync(job.Id, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsRetriableError(Exception ex)
|
||||
{
|
||||
var message = ex.Message.ToLowerInvariant();
|
||||
|
||||
var retriableErrors = new[]
|
||||
{
|
||||
"quota", "rate limit", "timeout", "service unavailable",
|
||||
"internal server error", "bad gateway", "gateway timeout",
|
||||
"network", "connection"
|
||||
};
|
||||
|
||||
return retriableErrors.Any(error => message.Contains(error)) ||
|
||||
ex is HttpRequestException ||
|
||||
ex is TimeoutException;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user