QueueJob - model and DB migration

WebAPI/AppDbContext.cs

@@ -3,17 +3,14 @@ using WebAPI.Models;
 
 namespace WebAPI;
 
-public class AppDbContext : DbContext
+public class AppDbContext(DbContextOptions<AppDbContext> options) : DbContext(options)
 {
     public DbSet<User> Users { get; init; }
     public DbSet<Layer> Layers { get; init; }
     public DbSet<Record> Records { get; init; }
     public DbSet<ProcessSource> ProcessSources { get; init; }
-    public DbSet<DataInbox> DataInbox { get; set; }
+    public DbSet<DataInbox> DataInbox { get; init; }
-
+    public DbSet<QueueJob> QueueJobs { get; init; }
-    public AppDbContext(DbContextOptions<AppDbContext> options) : base(options)
-    {
-    }
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {

WebAPI/Controllers/LayersController.cs

@@ -26,7 +26,7 @@ public class LayersController : Controller
         GoogleSheetsHelper googleSheetsHelper,
         GoogleDriveHelper googleDriveHelper,
         IConfiguration configuration
-        )
+    )
     {
         _db = db;
         if (googleSheetsHelper.Service is not null)
@@ -182,6 +182,83 @@ public class LayersController : Controller
         return Ok(true);
     }
 
+    [HttpGet]
+    [Route("AutoImportWithQueue/{apiKey}")]
+    [AllowAnonymous]
+    public async Task<IActionResult> AutoImportWithQueue(string apiKey)
+    {
+        if (Request.Host.Value != _configuration["apiLocalUrl"] || apiKey != _configuration["apiKey"])
+        {
+            return Unauthorized();
+        }
+
+        var importWorkerLayers = _db.Layers
+            .Include(x => x.Records)
+            .Where(x =>
+                x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
+                x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
+            )
+            .OrderBy(x => x.CreatedAt)
+            .ToList();
+
+        if (importWorkerLayers.Count == 0)
+        {
+            _logsController.AddEntry(new LogEntry
+            {
+                Title = "No Layers to import.",
+                Type = LogEntryType.Info,
+                LogType = LogType.Queue,
+                CreatedAt = DateTime.UtcNow
+            });
+            return Ok();
+        }
+
+        foreach (var importWorker in importWorkerLayers)
+        {
+            try
+            {
+                /*
+                await _queue.AddJob(new QueueJob
+                {
+                    LayerId = importWorker.Id,
+                    Type = JobType.ImportWorker,
+                });
+                */
+            }
+            catch (Exception e)
+            {
+                _logsController.AddEntry(new LogEntry
+                {
+                    Title = $"Error while adding job into queue (import): {importWorker.Name}, {importWorker.Id}",
+                    Type = LogEntryType.Error,
+                    LogType = LogType.Queue,
+                    Message = e.ToString(),
+                    CreatedAt = DateTime.UtcNow
+                });
+            }
+        }
+        return Ok();
+    }
+
+    [HttpGet]
+    [Route("ProcessQueue/{apiKey}")]
+    [AllowAnonymous]
+    public async Task<IActionResult> ProcessQueue(string apiKey)
+    {
+        /*
+        var allJobs = await _queue.GetJobs();
+        var importJobs = allJobs
+            .Where(x => x.Type == JobType.ImportWorker && x.Status == JobStatus.New);
+        
+        foreach (var job in importJobs)
+        {
+            job.Attempts = job.Attempts + 1;
+            //await _queue.UpdateJob(job);
+        }
+        */
+        return Ok();
+    }
+
     [HttpGet]
     [Route("AutoImport/{apiKey}")]
     [AllowAnonymous]
@@ -200,8 +277,8 @@ public class LayersController : Controller
         var importWorkerLayers = _db.Layers
             .Include(x => x.Records)
             .Where(x =>
-                    x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
+                x.Records!.Any(y => y.Code == "Type" && y.Desc1 == "ImportWorker") &&
-                    x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
+                x.Records!.Any(y => y.Code == "IsEnabled" && y.Desc1 == "True")
             )
             .OrderBy(x => x.CreatedAt)
             .ToList();
@@ -231,7 +308,7 @@ public class LayersController : Controller
                         var importer = new MorskaFk2Importer(_db, _googleSheetValues, this);
                         importer.Import(importWorker);
                         Thread.Sleep(5000); // be aware of GSheet API quota
-                        
+
                         _logsController.AddEntry(new LogEntry
                         {
                             Title = $"{importWorker.Name}, {importWorker.Id}",
@@ -665,9 +742,10 @@ public class LayersController : Controller
 
             if (!double.TryParse(data[1][i].ToString(), CultureInfo.GetCultureInfo("pl-PL"),
                     out var value) ||
-                double.Abs((double)(record.Value1-value)!) < 0.01) continue;
+                double.Abs((double)(record.Value1 - value)!) < 0.01) continue;
             isUpToDate = false;
         }
+
         foreach (var record in newestLayer.Records!)
         {
             if (data[0].Contains(record.Code))
@@ -678,6 +756,7 @@ public class LayersController : Controller
             WriteToConsole($"Code not found in GoogleSheet: {record.Code}");
             isUpToDate = false;
         }
+
         return isUpToDate;
     }
 }

WebAPI/Controllers/LogsController.cs

@@ -33,6 +33,7 @@ public class LogsController : Controller
             LogType.Process => "Process",
             LogType.PowerBi => "PowerBIAccess",
             LogType.DataInbox => "DataInbox",
+            LogType.Queue => "Queue",
             _ => "Other"
         };
         var response = _googleSheetValues.Get(_configuration["appLogsFile"], $"{type}!A:A").Execute();

WebAPI/DiunaBI-WebAPI.csproj

@@ -20,6 +20,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
+    <PackageReference Include="StackExchange.Redis" Version="2.8.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
     <PackageReference Include="System.Configuration.ConfigurationManager" Version="8.0.0" />
   </ItemGroup>
@@ -36,4 +37,8 @@
     <None Remove="dataProcessors\" />
     <None Remove="dataImporters\" />
   </ItemGroup>
+
+  <ItemGroup>
+    <Compile Remove="Controllers\QueueController.cs" />
+  </ItemGroup>
 </Project>

WebAPI/Migrations/20240825144443_QueueJobs.Designer.cs

@@ -0,0 +1,381 @@
+// <auto-generated />
+using System;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using WebAPI;
+
+#nullable disable
+
+namespace WebAPI.Migrations
+{
+    [DbContext(typeof(AppDbContext))]
+    [Migration("20240825144443_QueueJobs")]
+    partial class QueueJobs
+    {
+        /// <inheritdoc />
+        protected override void BuildTargetModel(ModelBuilder modelBuilder)
+        {
+#pragma warning disable 612, 618
+            modelBuilder
+                .HasAnnotation("ProductVersion", "8.0.6")
+                .HasAnnotation("Relational:MaxIdentifierLength", 128);
+
+            SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder);
+
+            modelBuilder.Entity("WebAPI.Models.DataInbox", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<string>("Data")
+                        .IsRequired()
+                        .HasMaxLength(2147483647)
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.Property<string>("Source")
+                        .IsRequired()
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("DataInbox");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.Layer", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("CreatedById")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<bool>("IsDeleted")
+                        .HasColumnType("bit");
+
+                    b.Property<DateTime>("ModifiedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("ModifiedById")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<string>("Name")
+                        .IsRequired()
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.Property<int>("Number")
+                        .HasColumnType("int");
+
+                    b.Property<Guid?>("ParentId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("int");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("CreatedById");
+
+                    b.HasIndex("ModifiedById");
+
+                    b.HasIndex("ParentId");
+
+                    b.ToTable("Layers");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
+                {
+                    b.Property<Guid>("LayerId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<Guid>("SourceId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.HasKey("LayerId", "SourceId");
+
+                    b.HasIndex("SourceId");
+
+                    b.ToTable("ProcessSources");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<int>("Attempts")
+                        .HasColumnType("int");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("LayerId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<string>("Message")
+                        .IsRequired()
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<DateTime>("ModifiedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("int");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("int");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("QueueJobs");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.Record", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<string>("Code")
+                        .IsRequired()
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("CreatedById")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<string>("Desc1")
+                        .HasMaxLength(1000)
+                        .HasColumnType("nvarchar(1000)");
+
+                    b.Property<bool>("IsDeleted")
+                        .HasColumnType("bit");
+
+                    b.Property<Guid>("LayerId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<DateTime>("ModifiedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("ModifiedById")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<double?>("Value1")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value10")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value11")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value12")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value13")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value14")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value15")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value16")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value17")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value18")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value19")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value2")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value20")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value21")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value22")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value23")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value24")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value25")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value26")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value27")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value28")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value29")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value3")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value30")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value31")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value32")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value4")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value5")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value6")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value7")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value8")
+                        .HasColumnType("float");
+
+                    b.Property<double?>("Value9")
+                        .HasColumnType("float");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("CreatedById");
+
+                    b.HasIndex("LayerId");
+
+                    b.HasIndex("ModifiedById");
+
+                    b.ToTable("Records");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.User", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<string>("Email")
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.Property<string>("UserName")
+                        .HasMaxLength(50)
+                        .HasColumnType("nvarchar(50)");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("Users");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.Layer", b =>
+                {
+                    b.HasOne("WebAPI.Models.User", "CreatedBy")
+                        .WithMany()
+                        .HasForeignKey("CreatedById")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("WebAPI.Models.User", "ModifiedBy")
+                        .WithMany()
+                        .HasForeignKey("ModifiedById")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("WebAPI.Models.Layer", "Parent")
+                        .WithMany()
+                        .HasForeignKey("ParentId");
+
+                    b.Navigation("CreatedBy");
+
+                    b.Navigation("ModifiedBy");
+
+                    b.Navigation("Parent");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.ProcessSource", b =>
+                {
+                    b.HasOne("WebAPI.Models.Layer", "Source")
+                        .WithMany()
+                        .HasForeignKey("SourceId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("Source");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.Record", b =>
+                {
+                    b.HasOne("WebAPI.Models.User", "CreatedBy")
+                        .WithMany()
+                        .HasForeignKey("CreatedById")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("WebAPI.Models.Layer", null)
+                        .WithMany("Records")
+                        .HasForeignKey("LayerId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("WebAPI.Models.User", "ModifiedBy")
+                        .WithMany()
+                        .HasForeignKey("ModifiedById")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("CreatedBy");
+
+                    b.Navigation("ModifiedBy");
+                });
+
+            modelBuilder.Entity("WebAPI.Models.Layer", b =>
+                {
+                    b.Navigation("Records");
+                });
+#pragma warning restore 612, 618
+        }
+    }
+}

WebAPI/Migrations/20240825144443_QueueJobs.cs

@@ -0,0 +1,40 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace WebAPI.Migrations
+{
+    /// <inheritdoc />
+    public partial class QueueJobs : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "QueueJobs",
+                columns: table => new
+                {
+                    Id = table.Column<Guid>(type: "uniqueidentifier", nullable: false),
+                    LayerId = table.Column<Guid>(type: "uniqueidentifier", nullable: false),
+                    Attempts = table.Column<int>(type: "int", nullable: false),
+                    Status = table.Column<int>(type: "int", nullable: false),
+                    Type = table.Column<int>(type: "int", nullable: false),
+                    Message = table.Column<string>(type: "nvarchar(max)", nullable: false),
+                    CreatedAt = table.Column<DateTime>(type: "datetime2", nullable: false),
+                    ModifiedAt = table.Column<DateTime>(type: "datetime2", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_QueueJobs", x => x.Id);
+                });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "QueueJobs");
+        }
+    }
+}

WebAPI/Migrations/AppDbContextModelSnapshot.cs

@@ -112,6 +112,39 @@ namespace WebAPI.Migrations
                     b.ToTable("ProcessSources");
                 });
 
+            modelBuilder.Entity("WebAPI.Models.QueueJob", b =>
+                {
+                    b.Property<Guid>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<int>("Attempts")
+                        .HasColumnType("int");
+
+                    b.Property<DateTime>("CreatedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<Guid>("LayerId")
+                        .HasColumnType("uniqueidentifier");
+
+                    b.Property<string>("Message")
+                        .IsRequired()
+                        .HasColumnType("nvarchar(max)");
+
+                    b.Property<DateTime>("ModifiedAt")
+                        .HasColumnType("datetime2");
+
+                    b.Property<int>("Status")
+                        .HasColumnType("int");
+
+                    b.Property<int>("Type")
+                        .HasColumnType("int");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("QueueJobs");
+                });
+
             modelBuilder.Entity("WebAPI.Models.Record", b =>
                 {
                     b.Property<Guid>("Id")

WebAPI/Models/LogEntry.cs

@@ -11,7 +11,8 @@ public enum LogType {
     Backup,
     Process,
     PowerBi,
-    DataInbox
+    DataInbox,
+    Queue
 }
 public class LogEntry
 {

WebAPI/Models/QueueJob.cs

@@ -0,0 +1,29 @@
+using System.ComponentModel.DataAnnotations;
+using System.Security.Cryptography.X509Certificates;
+
+namespace WebAPI.Models;
+
+public enum JobStatus
+{
+    New,
+    Failed,
+    Success
+}
+
+public enum JobType
+{
+    ImportWorker,
+    ProcessWorker
+}
+
+public class QueueJob
+{
+    [Key] public Guid Id { get; set; }
+    [Required] public Guid LayerId { get; set; }
+    [Required] public int Attempts { get; set; }
+    [Required] public JobStatus Status { get; set; } = JobStatus.New;
+    [Required] public JobType Type { get; set; } = JobType.ImportWorker;
+    public string Message { get; set; } = string.Empty;
+    [Required] public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+    [Required] public DateTime ModifiedAt { get; set; } = DateTime.UtcNow;
+}