JobQueue for import layers
This commit is contained in:
@@ -21,14 +21,4 @@ public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(op
|
|||||||
x.SourceId
|
x.SourceId
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private static readonly LoggerFactory MyLoggerFactory =
|
|
||||||
new(new[] {
|
|
||||||
new Microsoft.Extensions.Logging.Debug.DebugLoggerProvider()
|
|
||||||
});
|
|
||||||
|
|
||||||
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
|
|
||||||
{
|
|
||||||
optionsBuilder.UseLoggerFactory(MyLoggerFactory);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -11,17 +11,17 @@ public class DesignTimeDbContextFactory : IDesignTimeDbContextFactory<AppDbConte
|
|||||||
public AppDbContext CreateDbContext(string[] args)
|
public AppDbContext CreateDbContext(string[] args)
|
||||||
{
|
{
|
||||||
var configuration = new ConfigurationBuilder()
|
var configuration = new ConfigurationBuilder()
|
||||||
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI")) // ✅ Poprawna ścieżka
|
.SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "../DiunaBI.WebAPI"))
|
||||||
.AddJsonFile("appsettings.json", optional: false)
|
.AddJsonFile("appsettings.json", optional: false)
|
||||||
.AddJsonFile("appsettings.Development.json", optional: true)
|
.AddJsonFile("appsettings.Development.json", optional: true)
|
||||||
.Build();
|
.Build();
|
||||||
|
|
||||||
var optionsBuilder = new DbContextOptionsBuilder<AppDbContext>();
|
var optionsBuilder = new DbContextOptionsBuilder<AppDbContext>();
|
||||||
|
|
||||||
var connectionString = configuration.GetConnectionString("DefaultConnection");
|
var connectionString = configuration.GetConnectionString("SQLDatabase");
|
||||||
if (string.IsNullOrEmpty(connectionString))
|
if (string.IsNullOrEmpty(connectionString))
|
||||||
{
|
{
|
||||||
throw new InvalidOperationException("Connection string 'DefaultConnection' not found in appsettings.json");
|
throw new InvalidOperationException("Connection string 'SQLDatabase' not found in appsettings.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
optionsBuilder.UseSqlServer(connectionString);
|
optionsBuilder.UseSqlServer(connectionString);
|
||||||
|
|||||||
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
415
src/Backend/DiunaBI.Core/Database/Migrations/20250607084540_QueueJobRefactor.Designer.cs
generated
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
// <auto-generated />
|
||||||
|
using System;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Metadata;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Migrations
|
||||||
|
{
|
||||||
|
[DbContext(typeof(AppDbContext))]
|
||||||
|
[Migration("20250607084540_QueueJobRefactor")]
|
||||||
|
partial class QueueJobRefactor
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void BuildTargetModel(ModelBuilder modelBuilder)
|
||||||
|
{
|
||||||
|
#pragma warning disable 612, 618
|
||||||
|
modelBuilder
|
||||||
|
.HasAnnotation("ProductVersion", "8.0.0")
|
||||||
|
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||||
|
|
||||||
|
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("Data")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(2147483647)
|
||||||
|
.HasColumnType("nvarchar(max)");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<string>("Source")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("DataInbox");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<bool>("IsCancelled")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<bool>("IsDeleted")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Name")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<int>("Number")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<Guid?>("ParentId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("Type")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.HasIndex("CreatedById");
|
||||||
|
|
||||||
|
b.HasIndex("ModifiedById");
|
||||||
|
|
||||||
|
b.HasIndex("ParentId");
|
||||||
|
|
||||||
|
b.ToTable("Layers");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<Guid>("SourceId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.HasKey("LayerId", "SourceId");
|
||||||
|
|
||||||
|
b.HasIndex("SourceId");
|
||||||
|
|
||||||
|
b.ToTable("ProcessSources");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("CompletedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("JobType")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("LastAttemptAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("LastError")
|
||||||
|
.HasMaxLength(1000)
|
||||||
|
.HasColumnType("nvarchar(1000)");
|
||||||
|
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("LayerName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(200)
|
||||||
|
.HasColumnType("nvarchar(200)");
|
||||||
|
|
||||||
|
b.Property<int>("MaxRetries")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("PluginName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(100)
|
||||||
|
.HasColumnType("nvarchar(100)");
|
||||||
|
|
||||||
|
b.Property<int>("Priority")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("RetryCount")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("QueueJobs");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Code")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("Desc1")
|
||||||
|
.HasMaxLength(10000)
|
||||||
|
.HasColumnType("nvarchar(max)");
|
||||||
|
|
||||||
|
b.Property<bool>("IsDeleted")
|
||||||
|
.HasColumnType("bit");
|
||||||
|
|
||||||
|
b.Property<Guid>("LayerId")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("ModifiedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<double?>("Value1")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value10")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value11")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value12")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value13")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value14")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value15")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value16")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value17")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value18")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value19")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value2")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value20")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value21")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value22")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value23")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value24")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value25")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value26")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value27")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value28")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value29")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value3")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value30")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value31")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value32")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value4")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value5")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value6")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value7")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value8")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.Property<double?>("Value9")
|
||||||
|
.HasColumnType("float");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.HasIndex("CreatedById");
|
||||||
|
|
||||||
|
b.HasIndex("LayerId");
|
||||||
|
|
||||||
|
b.HasIndex("ModifiedById");
|
||||||
|
|
||||||
|
b.ToTable("Records");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||||
|
{
|
||||||
|
b.Property<Guid>("Id")
|
||||||
|
.ValueGeneratedOnAdd()
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("Email")
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.Property<string>("UserName")
|
||||||
|
.HasMaxLength(50)
|
||||||
|
.HasColumnType("nvarchar(50)");
|
||||||
|
|
||||||
|
b.HasKey("Id");
|
||||||
|
|
||||||
|
b.ToTable("Users");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("CreatedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ModifiedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ParentId");
|
||||||
|
|
||||||
|
b.Navigation("CreatedBy");
|
||||||
|
|
||||||
|
b.Navigation("ModifiedBy");
|
||||||
|
|
||||||
|
b.Navigation("Parent");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("SourceId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.Navigation("Source");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
|
{
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("CreatedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||||
|
.WithMany("Records")
|
||||||
|
.HasForeignKey("LayerId")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
|
.WithMany()
|
||||||
|
.HasForeignKey("ModifiedById")
|
||||||
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
|
.IsRequired();
|
||||||
|
|
||||||
|
b.Navigation("CreatedBy");
|
||||||
|
|
||||||
|
b.Navigation("ModifiedBy");
|
||||||
|
});
|
||||||
|
|
||||||
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
|
{
|
||||||
|
b.Navigation("Records");
|
||||||
|
});
|
||||||
|
#pragma warning restore 612, 618
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
using System;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Migrations
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
public partial class QueueJobRefactor : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "Message",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Type",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "RetryCount");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "ModifiedAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "ModifiedAtUtc");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Attempts",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Priority");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "CompletedAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "CreatedAtUtc",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<Guid>(
|
||||||
|
name: "CreatedById",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "uniqueidentifier",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<int>(
|
||||||
|
name: "JobType",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "int",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: 0);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<DateTime>(
|
||||||
|
name: "LastAttemptAt",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "datetime2",
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "LastError",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(1000)",
|
||||||
|
maxLength: 1000,
|
||||||
|
nullable: true);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "LayerName",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(200)",
|
||||||
|
maxLength: 200,
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<int>(
|
||||||
|
name: "MaxRetries",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "int",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: 0);
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<Guid>(
|
||||||
|
name: "ModifiedById",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "uniqueidentifier",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: new Guid("00000000-0000-0000-0000-000000000000"));
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "PluginName",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(100)",
|
||||||
|
maxLength: 100,
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CompletedAt",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CreatedAtUtc",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "CreatedById",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "JobType",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LastAttemptAt",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LastError",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "LayerName",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "MaxRetries",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "ModifiedById",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.DropColumn(
|
||||||
|
name: "PluginName",
|
||||||
|
table: "QueueJobs");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "RetryCount",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Type");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "Priority",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "Attempts");
|
||||||
|
|
||||||
|
migrationBuilder.RenameColumn(
|
||||||
|
name: "ModifiedAtUtc",
|
||||||
|
table: "QueueJobs",
|
||||||
|
newName: "ModifiedAt");
|
||||||
|
|
||||||
|
migrationBuilder.AddColumn<string>(
|
||||||
|
name: "Message",
|
||||||
|
table: "QueueJobs",
|
||||||
|
type: "nvarchar(max)",
|
||||||
|
nullable: false,
|
||||||
|
defaultValue: "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,11 +1,10 @@
|
|||||||
// <auto-generated />
|
// <auto-generated />
|
||||||
using System;
|
using System;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
using Microsoft.EntityFrameworkCore.Metadata;
|
using Microsoft.EntityFrameworkCore.Metadata;
|
||||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||||
using DiunaBI.Core.Models;
|
|
||||||
using DiunaBI.Core.Database.Context;
|
|
||||||
|
|
||||||
#nullable disable
|
#nullable disable
|
||||||
|
|
||||||
@@ -18,12 +17,12 @@ namespace DiunaBI.Core.Migrations
|
|||||||
{
|
{
|
||||||
#pragma warning disable 612, 618
|
#pragma warning disable 612, 618
|
||||||
modelBuilder
|
modelBuilder
|
||||||
.HasAnnotation("ProductVersion", "9.0.0")
|
.HasAnnotation("ProductVersion", "8.0.0")
|
||||||
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
.HasAnnotation("Relational:MaxIdentifierLength", 128);
|
||||||
|
|
||||||
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.DataInbox", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.DataInbox", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -52,7 +51,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("DataInbox");
|
b.ToTable("DataInbox");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -101,7 +100,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Layers");
|
b.ToTable("Layers");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("LayerId")
|
b.Property<Guid>("LayerId")
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
@@ -116,32 +115,63 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("ProcessSources");
|
b.ToTable("ProcessSources");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.QueueJob", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
b.Property<int>("Attempts")
|
b.Property<DateTime?>("CompletedAt")
|
||||||
.HasColumnType("int");
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
b.Property<DateTime>("CreatedAt")
|
b.Property<DateTime>("CreatedAt")
|
||||||
.HasColumnType("datetime2");
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<DateTime>("CreatedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("CreatedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<int>("JobType")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<DateTime?>("LastAttemptAt")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<string>("LastError")
|
||||||
|
.HasMaxLength(1000)
|
||||||
|
.HasColumnType("nvarchar(1000)");
|
||||||
|
|
||||||
b.Property<Guid>("LayerId")
|
b.Property<Guid>("LayerId")
|
||||||
.HasColumnType("uniqueidentifier");
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
b.Property<string>("Message")
|
b.Property<string>("LayerName")
|
||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("nvarchar(max)");
|
.HasMaxLength(200)
|
||||||
|
.HasColumnType("nvarchar(200)");
|
||||||
|
|
||||||
b.Property<DateTime>("ModifiedAt")
|
b.Property<int>("MaxRetries")
|
||||||
.HasColumnType("datetime2");
|
|
||||||
|
|
||||||
b.Property<int>("Status")
|
|
||||||
.HasColumnType("int");
|
.HasColumnType("int");
|
||||||
|
|
||||||
b.Property<int>("Type")
|
b.Property<DateTime>("ModifiedAtUtc")
|
||||||
|
.HasColumnType("datetime2");
|
||||||
|
|
||||||
|
b.Property<Guid>("ModifiedById")
|
||||||
|
.HasColumnType("uniqueidentifier");
|
||||||
|
|
||||||
|
b.Property<string>("PluginName")
|
||||||
|
.IsRequired()
|
||||||
|
.HasMaxLength(100)
|
||||||
|
.HasColumnType("nvarchar(100)");
|
||||||
|
|
||||||
|
b.Property<int>("Priority")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("RetryCount")
|
||||||
|
.HasColumnType("int");
|
||||||
|
|
||||||
|
b.Property<int>("Status")
|
||||||
.HasColumnType("int");
|
.HasColumnType("int");
|
||||||
|
|
||||||
b.HasKey("Id");
|
b.HasKey("Id");
|
||||||
@@ -149,7 +179,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("QueueJobs");
|
b.ToTable("QueueJobs");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -289,7 +319,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Records");
|
b.ToTable("Records");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.User", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.User", b =>
|
||||||
{
|
{
|
||||||
b.Property<Guid>("Id")
|
b.Property<Guid>("Id")
|
||||||
.ValueGeneratedOnAdd()
|
.ValueGeneratedOnAdd()
|
||||||
@@ -311,21 +341,21 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.ToTable("Users");
|
b.ToTable("Users");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("CreatedById")
|
.HasForeignKey("CreatedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ModifiedById")
|
.HasForeignKey("ModifiedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.Layer", "Parent")
|
b.HasOne("DiunaBI.Core.Models.Layer", "Parent")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ParentId");
|
.HasForeignKey("ParentId");
|
||||||
|
|
||||||
@@ -336,9 +366,9 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("Parent");
|
b.Navigation("Parent");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.ProcessSource", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.Layer", "Source")
|
b.HasOne("DiunaBI.Core.Models.Layer", "Source")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("SourceId")
|
.HasForeignKey("SourceId")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
@@ -347,21 +377,21 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("Source");
|
b.Navigation("Source");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Record", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Record", b =>
|
||||||
{
|
{
|
||||||
b.HasOne("WebAPI.Models.User", "CreatedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "CreatedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("CreatedById")
|
.HasForeignKey("CreatedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.Layer", null)
|
b.HasOne("DiunaBI.Core.Models.Layer", null)
|
||||||
.WithMany("Records")
|
.WithMany("Records")
|
||||||
.HasForeignKey("LayerId")
|
.HasForeignKey("LayerId")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
.IsRequired();
|
.IsRequired();
|
||||||
|
|
||||||
b.HasOne("WebAPI.Models.User", "ModifiedBy")
|
b.HasOne("DiunaBI.Core.Models.User", "ModifiedBy")
|
||||||
.WithMany()
|
.WithMany()
|
||||||
.HasForeignKey("ModifiedById")
|
.HasForeignKey("ModifiedById")
|
||||||
.OnDelete(DeleteBehavior.Cascade)
|
.OnDelete(DeleteBehavior.Cascade)
|
||||||
@@ -372,7 +402,7 @@ namespace DiunaBI.Core.Migrations
|
|||||||
b.Navigation("ModifiedBy");
|
b.Navigation("ModifiedBy");
|
||||||
});
|
});
|
||||||
|
|
||||||
modelBuilder.Entity("WebAPI.Models.Layer", b =>
|
modelBuilder.Entity("DiunaBI.Core.Models.Layer", b =>
|
||||||
{
|
{
|
||||||
b.Navigation("Records");
|
b.Navigation("Records");
|
||||||
});
|
});
|
||||||
|
|||||||
202
src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs
Normal file
202
src/Backend/DiunaBI.Core/Services/JobQueueProcessor.cs
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using DiunaBI.Core.Models;
|
||||||
|
using DiunaBI.Core.Interfaces;
|
||||||
|
using DiunaBI.Core.Database.Context;
|
||||||
|
using System;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Net.Http;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace DiunaBI.Core.Services;
|
||||||
|
|
||||||
|
public class JobQueueProcessor : BackgroundService
|
||||||
|
{
|
||||||
|
private readonly IServiceProvider _serviceProvider;
|
||||||
|
private readonly ILogger<JobQueueProcessor> _logger;
|
||||||
|
|
||||||
|
public JobQueueProcessor(
|
||||||
|
IServiceProvider serviceProvider,
|
||||||
|
ILogger<JobQueueProcessor> logger)
|
||||||
|
{
|
||||||
|
_serviceProvider = serviceProvider;
|
||||||
|
_logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Started");
|
||||||
|
|
||||||
|
while (!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var scope = _serviceProvider.CreateScope();
|
||||||
|
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||||
|
|
||||||
|
// First process all imports (they run sequentially due to Google Sheets API limits)
|
||||||
|
await ProcessJobType(queueService, JobType.Import, maxConcurrency: 1, stoppingToken);
|
||||||
|
|
||||||
|
// Then process processors (can run in parallel within same priority)
|
||||||
|
await ProcessJobType(queueService, JobType.Process, maxConcurrency: 3, stoppingToken);
|
||||||
|
|
||||||
|
// Wait before next cycle
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
|
||||||
|
}
|
||||||
|
catch (OperationCanceledException)
|
||||||
|
{
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Cancellation requested");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "JobQueueProcessor: Unexpected error in queue processor");
|
||||||
|
await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Stopped");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task ProcessJobType(IJobQueueService queueService, JobType jobType, int maxConcurrency, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
var runningJobs = await queueService.GetRunningJobsCountAsync(jobType);
|
||||||
|
|
||||||
|
// Don't start new jobs if we're at max concurrency
|
||||||
|
if (runningJobs >= maxConcurrency)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var job = await queueService.DequeueJobAsync(jobType);
|
||||||
|
if (job == null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Processing {JobType} job {JobId} for layer {LayerName} (attempt {RetryCount}/{MaxRetries}, priority {Priority})",
|
||||||
|
job.JobType, job.Id, job.LayerName, job.RetryCount + 1, job.MaxRetries, job.Priority);
|
||||||
|
|
||||||
|
// Process job asynchronously to allow parallel processing of processors
|
||||||
|
_ = Task.Run(async () =>
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await ProcessJobAsync(job, cancellationToken);
|
||||||
|
|
||||||
|
// Add delay between imports to respect Google Sheets API limits
|
||||||
|
if (job.JobType == JobType.Import)
|
||||||
|
{
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "JobQueueProcessor: Error in background job processing for {JobType} job {JobId}",
|
||||||
|
job.JobType, job.Id);
|
||||||
|
}
|
||||||
|
}, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task ProcessJobAsync(QueueJob job, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var scope = _serviceProvider.CreateScope();
|
||||||
|
var dbContext = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||||
|
var pluginManager = scope.ServiceProvider.GetRequiredService<PluginManager>();
|
||||||
|
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||||
|
|
||||||
|
// Get the layer with records
|
||||||
|
var layer = await dbContext.Layers
|
||||||
|
.Include(x => x.Records)
|
||||||
|
.FirstOrDefaultAsync(x => x.Id == job.LayerId && !x.IsDeleted, cancellationToken);
|
||||||
|
|
||||||
|
if (layer == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("JobQueueProcessor: Layer {LayerId} not found, marking job as failed", job.LayerId);
|
||||||
|
await queueService.MarkJobFailedAsync(job.Id, "Layer not found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process based on job type
|
||||||
|
switch (job.JobType)
|
||||||
|
{
|
||||||
|
case JobType.Import:
|
||||||
|
var importer = pluginManager.GetImporter(job.PluginName);
|
||||||
|
if (importer == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("JobQueueProcessor: Importer {PluginName} not found, marking job as failed", job.PluginName);
|
||||||
|
await queueService.MarkJobFailedAsync(job.Id, $"Importer {job.PluginName} not found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Executing import for layer {LayerName} with plugin {PluginName}",
|
||||||
|
layer.Name, job.PluginName);
|
||||||
|
|
||||||
|
importer.Import(layer);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case JobType.Process:
|
||||||
|
var processor = pluginManager.GetProcessor(job.PluginName);
|
||||||
|
if (processor == null)
|
||||||
|
{
|
||||||
|
_logger.LogWarning("JobQueueProcessor: Processor {PluginName} not found, marking job as failed", job.PluginName);
|
||||||
|
await queueService.MarkJobFailedAsync(job.Id, $"Processor {job.PluginName} not found");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Executing process for layer {LayerName} with plugin {PluginName}",
|
||||||
|
layer.Name, job.PluginName);
|
||||||
|
|
||||||
|
processor.Process(layer);
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new ArgumentOutOfRangeException(nameof(job.JobType), job.JobType, "Unknown job type");
|
||||||
|
}
|
||||||
|
|
||||||
|
await queueService.MarkJobCompletedAsync(job.Id);
|
||||||
|
|
||||||
|
_logger.LogInformation("JobQueueProcessor: Successfully completed {JobType} for layer {LayerName}",
|
||||||
|
job.JobType, layer.Name);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "JobQueueProcessor: Error processing {JobType} job {JobId} for layer {LayerName}",
|
||||||
|
job.JobType, job.Id, job.LayerName);
|
||||||
|
|
||||||
|
using var scope = _serviceProvider.CreateScope();
|
||||||
|
var queueService = scope.ServiceProvider.GetRequiredService<IJobQueueService>();
|
||||||
|
|
||||||
|
// Check if it's a retriable error
|
||||||
|
if (IsRetriableError(ex))
|
||||||
|
{
|
||||||
|
await queueService.MarkJobForRetryAsync(job.Id, ex.Message);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
await queueService.MarkJobFailedAsync(job.Id, ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsRetriableError(Exception ex)
|
||||||
|
{
|
||||||
|
var message = ex.Message.ToLowerInvariant();
|
||||||
|
|
||||||
|
var retriableErrors = new[]
|
||||||
|
{
|
||||||
|
"quota", "rate limit", "timeout", "service unavailable",
|
||||||
|
"internal server error", "bad gateway", "gateway timeout",
|
||||||
|
"network", "connection"
|
||||||
|
};
|
||||||
|
|
||||||
|
return retriableErrors.Any(error => message.Contains(error)) ||
|
||||||
|
ex is HttpRequestException ||
|
||||||
|
ex is TimeoutException;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,7 +7,7 @@ using Microsoft.EntityFrameworkCore;
|
|||||||
using DiunaBI.Core.Models;
|
using DiunaBI.Core.Models;
|
||||||
using DiunaBI.Core.Database.Context;
|
using DiunaBI.Core.Database.Context;
|
||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Cloud.Firestore;
|
using DiunaBI.Core.Interfaces;
|
||||||
|
|
||||||
namespace DiunaBI.WebAPI.Controllers;
|
namespace DiunaBI.WebAPI.Controllers;
|
||||||
|
|
||||||
@@ -21,6 +21,7 @@ public class LayersController : Controller
|
|||||||
private readonly IConfiguration _configuration;
|
private readonly IConfiguration _configuration;
|
||||||
private readonly PluginManager _pluginManager;
|
private readonly PluginManager _pluginManager;
|
||||||
private readonly ILogger<LayersController> _logger;
|
private readonly ILogger<LayersController> _logger;
|
||||||
|
private readonly IJobQueueService _queueService;
|
||||||
|
|
||||||
public LayersController(
|
public LayersController(
|
||||||
AppDbContext db,
|
AppDbContext db,
|
||||||
@@ -28,7 +29,8 @@ public class LayersController : Controller
|
|||||||
GoogleDriveHelper googleDriveHelper,
|
GoogleDriveHelper googleDriveHelper,
|
||||||
IConfiguration configuration,
|
IConfiguration configuration,
|
||||||
PluginManager pluginManager,
|
PluginManager pluginManager,
|
||||||
ILogger<LayersController> logger
|
ILogger<LayersController> logger,
|
||||||
|
IJobQueueService queueService
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
_db = db;
|
_db = db;
|
||||||
@@ -37,6 +39,7 @@ public class LayersController : Controller
|
|||||||
_configuration = configuration;
|
_configuration = configuration;
|
||||||
_pluginManager = pluginManager;
|
_pluginManager = pluginManager;
|
||||||
_logger = logger;
|
_logger = logger;
|
||||||
|
_queueService = queueService;
|
||||||
}
|
}
|
||||||
|
|
||||||
[HttpGet]
|
[HttpGet]
|
||||||
@@ -677,56 +680,81 @@ public class LayersController : Controller
|
|||||||
[HttpGet]
|
[HttpGet]
|
||||||
[Route("GetImportWorkers")]
|
[Route("GetImportWorkers")]
|
||||||
[AllowAnonymous]
|
[AllowAnonymous]
|
||||||
public IActionResult GetImportWorkers()
|
public async Task<IActionResult> GetImportWorkers()
|
||||||
{
|
{
|
||||||
var importWorkerLayers = _db.Layers
|
try
|
||||||
.Include(x => x.Records)
|
|
||||||
.Where(x =>
|
|
||||||
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
|
||||||
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
|
|
||||||
&& x.Number == 7270
|
|
||||||
)
|
|
||||||
.OrderByDescending(x => x.CreatedAt)
|
|
||||||
.AsNoTracking()
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
foreach (var importWorker in importWorkerLayers)
|
|
||||||
{
|
{
|
||||||
_logger.LogDebug("GetImportWorkers: Found import worker layer {LayerName} ({LayerId})",
|
var importWorkerLayers = await _db.Layers
|
||||||
importWorker.Name, importWorker.Id);
|
.Include(x => x.Records)
|
||||||
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
|
.Where(x =>
|
||||||
if (pluginName != null)
|
x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
|
||||||
|
x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
|
||||||
|
&& x.Number == 5579
|
||||||
|
)
|
||||||
|
.OrderBy(x => x.CreatedAt)
|
||||||
|
.AsNoTracking()
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
_logger.LogInformation("GetImportWorkers: Found {LayerCount} import worker layers to queue",
|
||||||
|
importWorkerLayers.Count);
|
||||||
|
|
||||||
|
int queuedCount = 0;
|
||||||
|
|
||||||
|
foreach (var importWorker in importWorkerLayers)
|
||||||
{
|
{
|
||||||
|
var pluginName = importWorker.Records!.FirstOrDefault(x => x.Code == "Plugin")?.Desc1;
|
||||||
|
if (string.IsNullOrEmpty(pluginName))
|
||||||
|
{
|
||||||
|
_logger.LogWarning("GetImportWorkers: No plugin name found for layer {LayerName} ({LayerId}), skipping",
|
||||||
|
importWorker.Name, importWorker.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if plugin exists
|
||||||
var importer = _pluginManager.GetImporter(pluginName);
|
var importer = _pluginManager.GetImporter(pluginName);
|
||||||
if (importer == null)
|
if (importer == null)
|
||||||
{
|
{
|
||||||
_logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId})",
|
_logger.LogWarning("GetImportWorkers: Importer {PluginName} not found for layer {LayerName} ({LayerId}), skipping",
|
||||||
pluginName, importWorker.Name, importWorker.Id);
|
pluginName, importWorker.Name, importWorker.Id);
|
||||||
throw new Exception($"Importer {pluginName} not found for layer {importWorker.Name}");
|
continue;
|
||||||
}
|
}
|
||||||
try
|
|
||||||
|
var job = new QueueJob
|
||||||
{
|
{
|
||||||
_logger.LogInformation("GetImportWorkers: Starting import for layer {LayerName} ({LayerId}) with plugin {PluginName}",
|
LayerId = importWorker.Id,
|
||||||
importWorker.Name, importWorker.Id, pluginName);
|
LayerName = importWorker.Name ?? "Unknown",
|
||||||
importer.Import(importWorker);
|
PluginName = pluginName,
|
||||||
_logger.LogInformation("GetImportWorkers: Successfully imported layer {LayerName} ({LayerId})",
|
JobType = JobType.Import,
|
||||||
importWorker.Name, importWorker.Id);
|
Priority = 0, // All imports have same priority
|
||||||
}
|
MaxRetries = 5,
|
||||||
catch (Exception e)
|
CreatedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D"),
|
||||||
{
|
ModifiedById = Guid.Parse("F392209E-123E-4651-A5A4-0B1D6CF9FF9D")
|
||||||
_logger.LogError(e, "GetImportWorkers: Error importing layer {LayerName} ({LayerId}) with plugin {PluginName}",
|
};
|
||||||
importWorker.Name, importWorker.Id, pluginName);
|
|
||||||
throw;
|
await _queueService.EnqueueJobAsync(job);
|
||||||
}
|
queuedCount++;
|
||||||
}
|
|
||||||
else
|
_logger.LogDebug("GetImportWorkers: Queued import job for layer {LayerName} ({LayerId}) with plugin {PluginName}",
|
||||||
{
|
importWorker.Name, importWorker.Id, pluginName);
|
||||||
_logger.LogWarning("GetImportWorkers: No plugin name found for import worker layer {LayerName} ({LayerId})",
|
|
||||||
importWorker.Name, importWorker.Id);
|
|
||||||
throw new Exception($"No plugin name found for import worker layer {importWorker.Name}");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var totalQueueSize = await _queueService.GetQueueCountAsync();
|
||||||
|
|
||||||
|
_logger.LogInformation("GetImportWorkers: Successfully queued {QueuedCount} import jobs. Total queue size: {QueueSize}",
|
||||||
|
queuedCount, totalQueueSize);
|
||||||
|
|
||||||
|
return Ok(new {
|
||||||
|
Message = $"Queued {queuedCount} import jobs",
|
||||||
|
QueuedJobs = queuedCount,
|
||||||
|
TotalQueueSize = totalQueueSize,
|
||||||
|
SkippedLayers = importWorkerLayers.Count - queuedCount
|
||||||
|
});
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
_logger.LogError(e, "GetImportWorkers: Error queuing import workers");
|
||||||
|
return BadRequest(e.ToString());
|
||||||
}
|
}
|
||||||
return Ok();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
[HttpGet]
|
[HttpGet]
|
||||||
|
|||||||
@@ -10,20 +10,24 @@ using DiunaBI.Core.Database.Context;
|
|||||||
using DiunaBI.Core.Services;
|
using DiunaBI.Core.Services;
|
||||||
using Google.Apis.Sheets.v4;
|
using Google.Apis.Sheets.v4;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
|
using DiunaBI.Core.Interfaces;
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
// ✅ DODAJ SERILOG CONFIGURATION
|
// ✅ SERILOG TYLKO DLA PRODUKCJI
|
||||||
builder.Host.UseSerilog((context, configuration) =>
|
if (builder.Environment.IsProduction())
|
||||||
{
|
{
|
||||||
configuration
|
builder.Host.UseSerilog((context, configuration) =>
|
||||||
.ReadFrom.Configuration(context.Configuration)
|
{
|
||||||
.Enrich.FromLogContext()
|
configuration
|
||||||
.Enrich.WithProperty("Application", "DiunaBI")
|
.ReadFrom.Configuration(context.Configuration)
|
||||||
.Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown")
|
.Enrich.FromLogContext()
|
||||||
.Enrich.WithEnvironmentName()
|
.Enrich.WithProperty("Application", "DiunaBI")
|
||||||
.Enrich.WithMachineName();
|
.Enrich.WithProperty("Version", Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown")
|
||||||
});
|
.Enrich.WithEnvironmentName()
|
||||||
|
.Enrich.WithMachineName();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
|
var connectionString = builder.Configuration.GetConnectionString("SQLDatabase");
|
||||||
builder.Services.AddDbContext<AppDbContext>(x =>
|
builder.Services.AddDbContext<AppDbContext>(x =>
|
||||||
@@ -50,7 +54,6 @@ builder.Services.AddCors(options =>
|
|||||||
|
|
||||||
builder.Services.AddControllers();
|
builder.Services.AddControllers();
|
||||||
|
|
||||||
|
|
||||||
builder.Services.AddAuthentication(options =>
|
builder.Services.AddAuthentication(options =>
|
||||||
{
|
{
|
||||||
options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
|
options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
|
||||||
@@ -66,10 +69,13 @@ builder.Services.AddAuthentication(options =>
|
|||||||
ValidateIssuerSigningKey = true,
|
ValidateIssuerSigningKey = true,
|
||||||
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!))
|
IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(builder.Configuration["Secret"]!))
|
||||||
};
|
};
|
||||||
|
|
||||||
});
|
});
|
||||||
builder.Services.AddAuthentication();
|
builder.Services.AddAuthentication();
|
||||||
|
|
||||||
|
// Queue services
|
||||||
|
builder.Services.AddScoped<IJobQueueService, JobQueueService>();
|
||||||
|
builder.Services.AddHostedService<JobQueueProcessor>();
|
||||||
|
|
||||||
// Zarejestruj Google Sheets dependencies
|
// Zarejestruj Google Sheets dependencies
|
||||||
builder.Services.AddSingleton<GoogleSheetsHelper>();
|
builder.Services.AddSingleton<GoogleSheetsHelper>();
|
||||||
builder.Services.AddSingleton<GoogleDriveHelper>();
|
builder.Services.AddSingleton<GoogleDriveHelper>();
|
||||||
@@ -90,32 +96,46 @@ builder.Services.AddSingleton<PluginManager>();
|
|||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
|
|
||||||
app.UseSerilogRequestLogging(options =>
|
// ✅ SERILOG REQUEST LOGGING TYLKO DLA PRODUKCJI
|
||||||
|
if (app.Environment.IsProduction())
|
||||||
{
|
{
|
||||||
options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms";
|
app.UseSerilogRequestLogging(options =>
|
||||||
options.EnrichDiagnosticContext = (diagnosticContext, httpContext) =>
|
|
||||||
{
|
{
|
||||||
diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value);
|
options.MessageTemplate = "HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms";
|
||||||
diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme);
|
options.EnrichDiagnosticContext = (diagnosticContext, httpContext) =>
|
||||||
|
|
||||||
var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault();
|
|
||||||
if (!string.IsNullOrEmpty(userAgent))
|
|
||||||
{
|
{
|
||||||
diagnosticContext.Set("UserAgent", userAgent);
|
diagnosticContext.Set("RequestHost", httpContext.Request.Host.Value);
|
||||||
}
|
diagnosticContext.Set("RequestScheme", httpContext.Request.Scheme);
|
||||||
|
|
||||||
diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown");
|
var userAgent = httpContext.Request.Headers.UserAgent.FirstOrDefault();
|
||||||
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
|
if (!string.IsNullOrEmpty(userAgent))
|
||||||
};
|
{
|
||||||
});
|
diagnosticContext.Set("UserAgent", userAgent);
|
||||||
|
}
|
||||||
|
|
||||||
|
diagnosticContext.Set("RemoteIP", httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown");
|
||||||
|
diagnosticContext.Set("RequestContentType", httpContext.Request.ContentType ?? "none");
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
var pluginManager = app.Services.GetRequiredService<PluginManager>();
|
var pluginManager = app.Services.GetRequiredService<PluginManager>();
|
||||||
var executablePath = Assembly.GetExecutingAssembly().Location;
|
var executablePath = Assembly.GetExecutingAssembly().Location;
|
||||||
var executableDir = Path.GetDirectoryName(executablePath)!;
|
var executableDir = Path.GetDirectoryName(executablePath)!;
|
||||||
var pluginsPath = Path.Combine(executableDir, "Plugins");
|
var pluginsPath = Path.Combine(executableDir, "Plugins");
|
||||||
|
|
||||||
Log.Information("Starting DiunaBI application");
|
// ✅ RÓŻNE LOGGERY W ZALEŻNOŚCI OD ŚRODOWISKA
|
||||||
Log.Information("Loading plugins from: {PluginsPath}", pluginsPath);
|
if (app.Environment.IsProduction())
|
||||||
|
{
|
||||||
|
Log.Information("Starting DiunaBI application");
|
||||||
|
Log.Information("Loading plugins from: {PluginsPath}", pluginsPath);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var logger = app.Services.GetRequiredService<ILogger<Program>>();
|
||||||
|
logger.LogInformation("Starting DiunaBI application (Development)");
|
||||||
|
logger.LogInformation("Loading plugins from: {PluginsPath}", pluginsPath);
|
||||||
|
}
|
||||||
|
|
||||||
pluginManager.LoadPluginsFromDirectory(pluginsPath);
|
pluginManager.LoadPluginsFromDirectory(pluginsPath);
|
||||||
|
|
||||||
@@ -145,5 +165,8 @@ app.MapControllers();
|
|||||||
|
|
||||||
app.Run();
|
app.Run();
|
||||||
|
|
||||||
// ✅ DODAJ CLEANUP
|
// ✅ SERILOG CLEANUP TYLKO DLA PRODUKCJI
|
||||||
Log.CloseAndFlush();
|
if (app.Environment.IsProduction())
|
||||||
|
{
|
||||||
|
Log.CloseAndFlush();
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user